level=info ts=2026-02-16T14:24:38.350435698Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T14:24:38.350474533Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T14:24:38.350491569Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T14:24:38.351478279Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T14:24:38.351867117Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T14:24:38.351949975Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
level=info ts=2026-02-16T14:24:38.352455746Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-ee30c144
ts=2026-02-16T14:24:38.352535135Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T14:24:38.352547901Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=debug ts=2026-02-16T14:24:38.352567065Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=info ts=2026-02-16T14:24:38.352672999Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-02-16T14:24:38.355613169Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:24:38.357210763Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T14:24:38.357414504Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:24:38.358391213Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T14:24:38.358410044Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T14:24:38.358408975Z
level=debug ts=2026-02-16T14:24:38.358424629Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T14:24:38.358439438Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T14:24:38.358453734Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-02-16T14:24:38.358457891Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-02-16T14:24:38.358457248Z
level=debug ts=2026-02-16T14:24:38.358461638Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-02-16T14:24:38.359889136Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T14:24:38.359974859Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
ts=2026-02-16T14:24:38.360876602Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T14:24:38.363012443Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-02-16T14:24:38.364580374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-02-16T14:24:38.36771014Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-02-16T14:24:38.370001597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-02-16T14:24:38.372516314Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
level=info ts=2026-02-16T14:24:38.372535754Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=19.865278ms
level=info ts=2026-02-16T14:24:38.372555467Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T14:24:38.375399393Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-02-16T14:24:38.375509197Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T14:24:38.375528521Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=info ts=2026-02-16T14:24:38.375548474Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T14:24:38.375546961Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-02-16T14:24:38.37559832Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T14:24:38.375616136Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-02-16T14:24:38.375607636Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T14:24:38.375631326Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T14:24:38.375577647Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-02-16T14:24:38.375731806Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=info ts=2026-02-16T14:24:38.375811123Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T14:24:38.37581191Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-02-16T14:24:38.375820951Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T14:24:38.375826931Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-02-16T14:24:38.375828529Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-02-16T14:24:38.375930912Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-02-16T14:24:38.375953611Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-02-16T14:24:38.375959122Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-02-16T14:24:38.376008349Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T14:24:38.376015036Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T14:24:38.376020515Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-02-16T14:24:38.376024945Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-02-16T14:24:38.376029026Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T14:24:38.376045492Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-02-16T14:24:38.37613555Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T14:24:38.377205127Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-02-16T14:24:38.378271553Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-02-16T14:24:38.380424938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-02-16T14:24:38.381965145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-02-16T14:24:38.383446603Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
level=info ts=2026-02-16T14:24:38.383465918Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=10.891983ms
level=info ts=2026-02-16T14:24:38.447199903Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=71.157396ms errors=false
level=info ts=2026-02-16T14:24:38.447236966Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-02-16T14:24:40.902935107Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=2.526891822s errors=false
level=info ts=2026-02-16T14:24:40.902977989Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T14:24:40.903022885Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=2.526980653s
level=info ts=2026-02-16T14:24:40.903113509Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T14:24:40.903120789Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T14:24:40.903153002Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=3.433098784s
level=info ts=2026-02-16T14:24:40.903178318Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-02-16T14:24:40.9032219Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T14:24:40.903247031Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T14:24:40.903251136Z caller=loki.go:599 msg="Loki started" startup_time=2.613513306s
ts=2026-02-16T14:24:42.457566111Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.129.0.66:7946"
ts=2026-02-16T14:24:43.92359891Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.128.0.73:7946"
ts=2026-02-16T14:24:44.458583861Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.130.0.68:7946"
ts=2026-02-16T14:24:44.459214194Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.128.0.72:7946"
ts=2026-02-16T14:24:44.459593293Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.128.0.73:7946"
level=warn ts=2026-02-16T14:24:45.903572793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:24:45.925950503Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.130.0.68:7946"
ts=2026-02-16T14:24:45.926288187Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.128.0.72:7946"
ts=2026-02-16T14:24:45.927829282Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-1a0c3718' from=10.129.0.66:7946"
ts=2026-02-16T14:24:46.238596476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:42532"
level=warn ts=2026-02-16T14:24:50.904098939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:24:55.903415589Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:24:57.128069253Z caller=logging.go:137 msg="GET /ready (503) 177.544µs"
level=warn ts=2026-02-16T14:25:00.90419985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:25:05.903842479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:07.12777836Z caller=logging.go:137 msg="GET /ready (503) 113.88µs"
level=warn ts=2026-02-16T14:25:10.903780667Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:10.90378434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T14:25:10.903819585Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T14:25:10.903829643Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=info ts=2026-02-16T14:25:10.903852942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:25:15.904267123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:18.560210718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:25:18.560218121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:25:18.560215928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:25:18.560273451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:25:18.560267565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:25:18.560285158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=info ts=2026-02-16T14:25:18.560295445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:25:18.560355605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="642 B" avg_comp="642 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:25:18.560361316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="438 B" avg_comp="438 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:25:18.56052332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="931 B" avg_comp="931 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:25:18.560557042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:25:18.560607223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:25:18.560976836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:25:18.560985508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:25:18.560996782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=info ts=2026-02-16T14:25:18.561051231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:25:18.561116722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:25:18.561158048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:25:18.561166861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="727 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:25:18.561218487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:25:18.561224291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:25:18.561251564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="668 kB" avg_uncomp="668 kB" idle=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:25:18.561288983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:25:18.561300499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:25:18.561315191Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:25:18.561352567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:18.567803332Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:18.567812387Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:25:18.567841735Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:25:18.56789156Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:25:18.56790036Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:25:18.567923472Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:18.567951058Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:25:18.567956482Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:25:18.567983743Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:25:18.568008751Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:25:18.568012366Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:25:18.56804086Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:25:18.568064336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:25:18.568068034Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:18.568123227Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:25:18.56812474Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:18.568124435Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:25:18.568172717Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:25:18.568178516Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T14:25:18.56818314Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:25:18.568222444Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:25:18.568216841Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:25:18.568233518Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:25:18.571431317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:25:18.57143317Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:25:18.861463874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:25:18.865530369Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:25:20.903322449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:25:25.903906691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:25:28.194207128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=info ts=2026-02-16T14:25:28.779415897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:28.786155312Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:25:29.298648526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:25:29.301630851Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:25:29.304670812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:25:29.307190898Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:25:29.457806682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:25:29.461050501Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:25:30.183567657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:25:30.186605136Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:25:30.797596068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:25:30.801691759Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:25:30.853905403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:25:30.857104611Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-02-16T14:25:30.904180573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:31.405243066Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:25:31.732521813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:25:31.735900957Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:25:32.173428942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:25:32.176790139Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:25:32.183855293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:25:32.186502842Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:25:32.739141139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:25:32.742592492Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:25:32.881425562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:25:32.884602759Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:25:33.006436521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:25:33.01319866Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:25:33.117463066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:25:33.121525374Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:25:33.55126664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:25:33.55779122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:25:34.310796439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:25:34.314385156Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:25:34.459214033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:25:34.462553381Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:25:35.134347107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:25:35.139092001Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=warn ts=2026-02-16T14:25:35.903583444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:36.347828398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:25:36.351048916Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:25:36.521669929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:25:36.526297254Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:25:37.1027894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:25:37.105764261Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:25:37.884577907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:25:37.88794087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:25:38.013344247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:25:38.016555091Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:25:38.356900628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:25:38.357953522Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:25:38.409592298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:25:38.412724606Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:25:38.471047775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:25:38.474105165Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:25:38.516224516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:25:38.520249179Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:25:40.904086103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:25:40.904102294Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:25:40.904127228Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:25:40.904138144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:25:45.904057208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:49.376435336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:25:49.379838074Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:25:50.431180134Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:25:50.903523043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:53.310429179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:25:53.313321417Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:25:53.966782775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:25:53.969763448Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T14:25:53.971908038Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:25:54.131272253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:25:54.135077338Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:25:54.73120975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:25:54.735052483Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:25:55.881135785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:25:55.884112854Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:25:55.90418574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:25:56.200791163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:25:56.203625863Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:25:56.559781705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:25:56.563511335Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:25:57.606803538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:25:57.610050301Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:25:57.669258877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:25:57.67208218Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
ts=2026-02-16T14:25:58.196645627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:25:58.974047209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:25:58.978623461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:25:59.596400763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:25:59.599574031Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:25:59.906322703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:25:59.909722462Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=warn ts=2026-02-16T14:26:00.903504505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:02.094959976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:26:02.097874579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:26:02.790147841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:26:02.792980702Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:26:02.796058434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:02.798551024Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:26:02.979680778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:26:02.982971919Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:26:03.417198024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:03.420671096Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:26:03.438661135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:26:03.446492684Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:26:05.23659778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:26:05.245468238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:26:05.682014358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:26:05.68500962Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:26:05.803210064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:38030"
level=warn ts=2026-02-16T14:26:05.903612424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:09.50632523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:26:09.509197782Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:26:10.904240134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:10.904236237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:26:10.90427075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:26:10.904282505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:26:12.407505515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:26:12.49774866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:26:12.500768249Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:26:14.120236892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:26:14.12426553Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:26:15.90362922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:16.61776383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:26:16.622228133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:26:17.92915463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:26:17.934264823Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=debug ts=2026-02-16T14:26:20.739704956Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:26:20.904158486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:26:25.904107144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:26:28.199140406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=warn ts=2026-02-16T14:26:30.903822085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:33.556848681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:26:33.560162882Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:26:35.493148991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:26:35.903448002Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:35.936637595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:26:35.941019066Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:26:38.29197095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:51908"
level=info ts=2026-02-16T14:26:38.357466402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:26:38.357913965Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:26:39.652660757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:26:39.656063141Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:26:40.16857894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:26:40.171837738Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:26:40.8778559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:26:40.881495952Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:26:40.903551491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:26:40.903564129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:26:40.903583971Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:26:40.903593016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:26:42.075562752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:26:42.078329088Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:26:45.204108865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:26:45.206953711Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T14:26:45.904136809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:26:46.251321701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43538"
level=info ts=2026-02-16T14:26:48.704058398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:26:48.707038586Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:26:49.108191763Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:26:50.21884462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:26:50.222467619Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:26:50.660852782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:26:50.664032642Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:26:50.83243596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:26:50.840262119Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-02-16T14:26:50.903448572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:51.079661705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:26:51.083158925Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:26:51.27520657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:26:51.278687072Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:26:51.988411141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:26:51.991972677Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-02-16T14:26:55.903544636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:26:56.10563245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:26:56.10905759Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:26:57.716448132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:26:57.720618061Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:26:58.10648947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:26:58.109495949Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:26:58.20152169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:26:59.168405132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:26:59.171546333Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:26:59.694730919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:26:59.697924595Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:26:59.781289228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:26:59.784456006Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:27:00.345735021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:27:00.349502319Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:27:00.371879133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:27:00.375082912Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:27:00.903348189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:02.993398695Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:27:02.995350664Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:27:04.937193296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:27:04.952682513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:27:05.904251947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:06.914011801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:27:06.91744723Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:27:10.904164761Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:27:10.904203626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:27:10.904214026Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:27:10.904236504Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:11.187681449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:27:11.191064467Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:27:11.333913831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:27:11.337810312Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:27:15.802164169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:27:15.805490647Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:27:15.903878068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:20.112090271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:27:20.115448724Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:27:20.903510553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:27:22.854305785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:27:24.862814496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:27:24.866864543Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:27:25.903949121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:27:28.203639232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=warn ts=2026-02-16T14:27:30.903949547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:32.394956381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:27:32.398396576Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:27:32.454712553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:27:32.457926939Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:27:35.219991781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:27:35.223451635Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:27:35.810250571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43904"
level=warn ts=2026-02-16T14:27:35.904132606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:36.091447877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:27:36.094783653Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:27:37.100783389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:27:37.104304308Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:27:38.295352504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:55584"
level=debug ts=2026-02-16T14:27:38.357436915Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:27:38.357471783Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:27:38.528286283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:27:38.531769924Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:27:38.851667596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:27:38.855097689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:27:38.994511936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:27:38.99808362Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:27:39.94509373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:27:39.951692878Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:27:40.903886399Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:27:40.90392439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:27:40.903935345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:27:40.903959606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:41.238067822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:27:41.241492879Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:27:44.067107529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:27:44.070129877Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=debug ts=2026-02-16T14:27:44.972663935Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:27:45.434969685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:27:45.437985963Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:27:45.904291576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:27:46.257740488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:43228"
level=info ts=2026-02-16T14:27:47.187420192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:27:47.190612881Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:27:48.69439009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:27:48.697332741Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:27:48.703357774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:27:48.706099693Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:27:48.928008438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:27:48.931037849Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:27:49.116532712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:27:49.119528346Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:27:49.727501811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:27:49.730586174Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:27:49.980405373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:27:49.986651606Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-02-16T14:27:50.736805902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:27:50.90334258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:54.285436792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:27:54.289027928Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:27:55.90435253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:27:56.111680019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:27:56.115147152Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
ts=2026-02-16T14:27:58.205986087Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=warn ts=2026-02-16T14:28:00.903349074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:04.382240469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:28:04.386836198Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T14:28:05.812491029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56244"
level=warn ts=2026-02-16T14:28:05.904088131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:07.367928973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:28:07.373327222Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:28:09.313571976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:28:09.316946899Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:28:10.90425003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:28:10.9042868Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:28:10.904295475Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:28:10.904344595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:10.925423399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:28:10.92905673Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:28:15.903519873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:16.251230619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:28:16.254708738Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=warn ts=2026-02-16T14:28:20.903395817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:21.387230633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:28:21.39598694Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:28:22.902798236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:28:22.906109724Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=debug ts=2026-02-16T14:28:22.953471349Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:28:25.903493972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:25.973822415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:28:25.976810479Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:28:27.956866042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:28:27.960036952Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T14:28:28.209008437Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=info ts=2026-02-16T14:28:28.428810784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:28:28.431848457Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:28:28.81631962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:28:28.819732186Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:28:29.453828306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:28:29.456956143Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:28:30.420757842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:28:30.423924727Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:28:30.50214507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:28:30.505158802Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=warn ts=2026-02-16T14:28:30.904296095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:31.35275636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:28:31.356240061Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:28:33.282208567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:28:33.285587454Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:28:33.746624246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:28:33.75321677Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-02-16T14:28:35.81494139Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58538"
level=warn ts=2026-02-16T14:28:35.903794269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:36.900392388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:28:36.903689923Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:28:37.460135313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:28:37.46411675Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T14:28:38.298925444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:60866"
level=info ts=2026-02-16T14:28:38.357747471Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:28:38.357763096Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:28:38.391339554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:28:38.394975693Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:28:38.466843422Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:28:40.826496693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:28:40.829888753Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:28:40.904125011Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:28:40.904143699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:28:40.904159851Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:28:40.904168368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:28:42.578421444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:28:42.581916013Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:28:45.242425976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:28:45.247295768Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:28:45.903345095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:46.649881047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:28:46.653524369Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:28:49.3088061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:28:49.313345701Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:28:50.82080226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:28:50.824080884Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:28:50.904343437Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:28:51.121241974Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:28:54.757034562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:28:54.760742429Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:28:55.146616927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:28:55.149721751Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:28:55.904151467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:28:56.438679196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:28:56.44245688Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:28:57.328426448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:28:57.331645422Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T14:28:58.212586859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=warn ts=2026-02-16T14:29:00.903838945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:03.342503046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:29:03.346847226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:29:05.90403849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:08.857336862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:29:08.864143684Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:29:10.903941254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:29:10.903935473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:10.903972427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:29:10.903984031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:29:13.163575952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:29:13.167083554Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:29:14.654062179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:29:14.658724275Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:29:15.856273535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:29:15.860175095Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=warn ts=2026-02-16T14:29:15.904265378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:15.956916254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:29:15.960409692Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:29:16.436958035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:29:17.266265985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:29:17.274550824Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:29:18.351803878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:29:18.359780456Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:29:19.63100873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:29:19.634995452Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:29:20.903880641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:20.953204949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:29:20.956965479Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:29:23.958862396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:29:23.961709872Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:29:24.472313486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:29:24.47540151Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:29:24.734932079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:29:24.738075672Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:29:25.904083337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:26.663426686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:29:26.666868567Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:29:27.797459499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="8.1 kB" avg_uncomp="8.1 kB" full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:29:27.810848539Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"application-service\"}"
ts=2026-02-16T14:29:28.214879232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=info ts=2026-02-16T14:29:30.62952978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:29:30.63573176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:29:30.903563895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:33.089505464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:29:33.094693091Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:29:33.746295328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:29:33.749078768Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:29:35.903996733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:37.576236547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:29:37.579561065Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:29:38.350072728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=1 total_comp="90 kB" avg_comp="90 kB" total_uncomp="851 kB" avg_uncomp="851 kB" full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:29:38.354016673Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-grafana\"}"
level=debug ts=2026-02-16T14:29:38.356681846Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:29:38.35772681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:29:40.903636787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:29:40.903670169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:29:40.903660663Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-02-16T14:29:40.903678912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:29:40.903671886Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:40.903820765Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-02-16T14:29:40.904092953Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=info ts=2026-02-16T14:29:41.672669432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:29:41.675716081Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:29:43.998829359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:29:44.002562908Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-02-16T14:29:44.366572568Z caller=checkpoint.go:378 msg="writing series" size="462 kB"
level=debug ts=2026-02-16T14:29:44.834330637Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:29:45.903516597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:47.833137621Z caller=checkpoint.go:378 msg="writing series" size="3.8 MB"
level=debug ts=2026-02-16T14:29:47.833174687Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.3 MB" series=3
level=info ts=2026-02-16T14:29:49.763414693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:29:49.766925537Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:29:50.534754811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:29:50.538771453Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=warn ts=2026-02-16T14:29:50.903841723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:29:51.28894991Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=debug ts=2026-02-16T14:29:52.281469491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:29:52.281550103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=be1a6f0d81e7ecea immediate=false num_chunks=1 total_comp="411 kB" avg_comp="411 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" idle=1 labels="{stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:29:52.292687823Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/be1a6f0d81e7ecea/19c66c0927b%3A19c66cd9056%3Aa31c6246?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pxgn\"}"
level=info ts=2026-02-16T14:29:52.350923078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:29:52.354027326Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:29:54.717681632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:29:54.722462692Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:29:54.751364682Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-02-16T14:29:55.527140835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:29:55.531827928Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:29:55.904053694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:29:56.158686786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:29:56.161882153Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:29:57.070682513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:29:57.074215031Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=debug ts=2026-02-16T14:29:58.212759804Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-02-16T14:29:58.21764973Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=warn ts=2026-02-16T14:30:00.903394491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:01.168594913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:30:01.172251201Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:30:01.674043053Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-02-16T14:30:02.942950045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="582 B" avg_comp="291 B" total_uncomp="983 B" avg_uncomp="491 B" idle=1 full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:30:02.947135401Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:30:04.977287782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:30:04.984981924Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:30:05.135273284Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-02-16T14:30:05.903607599Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:06.177582724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" idle=1 full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:30:06.181804537Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:30:07.098425924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:30:07.10313112Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
ts=2026-02-16T14:30:08.305517484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:43080"
level=debug ts=2026-02-16T14:30:08.596424795Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:30:08.969495248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="12 kB" avg_uncomp="6.0 kB" idle=1 full=1 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:30:08.972987481Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=info ts=2026-02-16T14:30:10.465424353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:30:10.470994224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=be1a6f0d81e7ecea immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" full=1 labels="{stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:30:10.471066455Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:30:10.474219432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:30:10.474899588Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/be1a6f0d81e7ecea/19c66c0927b%3A19c66cd9056%3Aa31c6246?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:30:10.477081765Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:30:10.647589744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:30:10.651644887Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:30:10.90421634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:30:10.904218057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:10.904249993Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:30:10.904260015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:30:12.05130574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:30:12.05465847Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-02-16T14:30:12.058889996Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-02-16T14:30:15.520191208Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-02-16T14:30:15.903500666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:30:16.272785564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:52800"
level=info ts=2026-02-16T14:30:16.357205545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:30:16.363545818Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:30:17.573302362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:30:17.576716743Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:30:17.683476248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:30:17.686640714Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:30:18.98391331Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-02-16T14:30:18.983938289Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.0 MB" series=9
level=warn ts=2026-02-16T14:30:20.903617291Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:22.094034696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:30:22.098736359Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=debug ts=2026-02-16T14:30:22.443183749Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
level=info ts=2026-02-16T14:30:22.863994406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:30:22.86763734Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:30:24.708184671Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:30:25.903521264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:25.904613044Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:30:26.711817705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:30:26.715724926Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:30:26.957552106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:30:26.961045405Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-02-16T14:30:28.219945507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:30:29.176793498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:30:29.179937449Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:30:29.366245104Z caller=checkpoint.go:378 msg="writing series" size="238 kB"
level=warn ts=2026-02-16T14:30:30.903629965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:30.932742459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" idle=1 full=1 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:30:30.936026347Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=debug ts=2026-02-16T14:30:32.827308267Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T14:30:33.325716392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:30:33.33022543Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-02-16T14:30:35.823462324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60478"
level=warn ts=2026-02-16T14:30:35.903737214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:35.921826717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:30:35.925295106Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=debug ts=2026-02-16T14:30:36.289350433Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=info ts=2026-02-16T14:30:38.147105435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="931 B" avg_comp="931 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:30:38.150328458Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:30:38.357466644Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:30:38.357470673Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=debug ts=2026-02-16T14:30:38.359803137Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-02-16T14:30:38.359857015Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1968057 n=1
level=debug ts=2026-02-16T14:30:38.35990529Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1968057 n=1
level=debug ts=2026-02-16T14:30:39.750697513Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-02-16T14:30:40.781038883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:30:40.785110878Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=info ts=2026-02-16T14:30:40.903258075Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:30:40.903298361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:30:40.903309327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:30:40.903363857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:43.211919546Z caller=checkpoint.go:378 msg="writing series" size="252 kB"
level=info ts=2026-02-16T14:30:43.662675529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:30:43.665936714Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:30:44.940718337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:30:44.943895289Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:30:45.892237322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="582 B" avg_comp="291 B" total_uncomp="983 B" avg_uncomp="491 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:30:45.89556275Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=warn ts=2026-02-16T14:30:45.903674501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:30:46.674019572Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=info ts=2026-02-16T14:30:47.931969502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:30:47.935461363Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:30:49.699402062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=be1a6f0d81e7ecea immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" full=1 labels="{stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:30:49.701979518Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/be1a6f0d81e7ecea/19c66c0927b%3A19c66cd9056%3Aa31c6246?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pxgn\"}"
level=debug ts=2026-02-16T14:30:50.136077208Z caller=checkpoint.go:378 msg="writing series" size="266 kB"
level=info ts=2026-02-16T14:30:50.345494146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:30:50.34859228Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:30:50.904284286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:52.431490224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:30:52.435856548Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:30:53.597223468Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=info ts=2026-02-16T14:30:54.480687309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:30:54.483897268Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=debug ts=2026-02-16T14:30:55.098243744Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:30:55.9043264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:30:56.184751034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:30:56.187989253Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=debug ts=2026-02-16T14:30:57.058237133Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:30:58.032835203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:30:58.036141462Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-02-16T14:30:58.222854381Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:30:58.305752914Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:30:59.022916007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:30:59.030119139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-02-16T14:31:00.52015725Z caller=checkpoint.go:378 msg="writing series" size="7.5 kB"
level=warn ts=2026-02-16T14:31:00.903407041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:01.486088181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="12 kB" avg_uncomp="6.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:31:01.489200351Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=debug ts=2026-02-16T14:31:03.983813194Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-02-16T14:31:03.983838554Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=13
level=info ts=2026-02-16T14:31:05.446989911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:31:05.452527365Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=warn ts=2026-02-16T14:31:05.904395923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:06.056272259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:31:06.060121496Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-02-16T14:31:07.443141517Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-02-16T14:31:07.800422776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:31:07.803965924Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:31:09.206642014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:31:09.209910996Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:31:09.679802837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:31:09.687460811Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:31:10.903839913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:31:10.903881162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:31:10.903892347Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:31:10.903893324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:10.905008807Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-02-16T14:31:12.048429504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:31:12.052675473Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-02-16T14:31:14.366254286Z caller=checkpoint.go:378 msg="writing series" size="860 kB"
level=debug ts=2026-02-16T14:31:14.366305275Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-02-16T14:31:14.375881173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:14.379183564Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:31:15.765363723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:31:15.769319724Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:31:15.90435443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:17.828186445Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=info ts=2026-02-16T14:31:18.427332099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" full=2 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:31:18.430764079Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:31:18.616330375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:31:18.61977821Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:31:20.903331116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:21.148358542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:31:21.151949811Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=info ts=2026-02-16T14:31:21.170172552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:31:21.17360994Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:31:21.173636435Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:21.173661498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:31:21.17663272Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=debug ts=2026-02-16T14:31:21.289851132Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=info ts=2026-02-16T14:31:21.86391831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:31:21.867416491Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:31:24.751072049Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-02-16T14:31:25.90335447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:28.212421478Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
ts=2026-02-16T14:31:28.225412697Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:31:28.40057171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:31:28.403783683Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:31:28.403806567Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:28.403829977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:31:28.406295486Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=debug ts=2026-02-16T14:31:30.228176972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:31:30.903821024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:31.491910006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:31:31.494959156Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=debug ts=2026-02-16T14:31:31.673520334Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=info ts=2026-02-16T14:31:32.73724819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=28119cab16f09aef immediate=false num_chunks=2 total_comp="11 kB" avg_comp="5.4 kB" total_uncomp="54 kB" avg_uncomp="27 kB" full=2 labels="{stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:31:32.740073611Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/28119cab16f09aef/19c66ba8137%3A19c66ba8138%3A75142b3b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"cert-manager-operator\"}"
level=error ts=2026-02-16T14:31:32.740096967Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:32.740122173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:32.742954675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:31:33.135440347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:33.138585371Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:33.138610109Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:33.138635073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:33.141125088Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:31:34.788498433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:34.791526536Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:34.791549527Z caller=flush.go:240 component=ingester loop=13 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:34.791572521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:34.793934927Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=debug ts=2026-02-16T14:31:35.135868623Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-02-16T14:31:35.903366174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:36.027076603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:31:36.032766896Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:31:36.032796308Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:36.032820901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:31:36.035574835Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:31:38.357596418Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:31:38.357616551Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=debug ts=2026-02-16T14:31:38.596766143Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-02-16T14:31:39.111320941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:31:39.114528808Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:31:39.114551084Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:39.114573227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:31:39.117082163Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:31:40.903757036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:31:40.903793079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:31:40.903801806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:31:40.903808834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:42.058468407Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=debug ts=2026-02-16T14:31:42.864219152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:31:42.866250992Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:31:43.856561458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="991 B" avg_comp="495 B" total_uncomp="983 B" avg_uncomp="491 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:31:43.860120834Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:31:43.860144159Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:43.860167097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="504 B" total_uncomp="983 B" avg_uncomp="491 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:31:43.862747533Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:31:44.24514949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=be1a6f0d81e7ecea immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" full=1 labels="{stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:31:44.249496412Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/be1a6f0d81e7ecea/19c66c0927b%3A19c66cd9056%3Aa31c6246?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pxgn\"}"
level=debug ts=2026-02-16T14:31:45.520476688Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-02-16T14:31:45.631922072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:31:45.635708979Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:31:45.635733388Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:45.635756591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:31:45.638519448Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=warn ts=2026-02-16T14:31:45.903431898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:31:46.157185882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:31:46.160458436Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:31:46.160478328Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:46.160498817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:31:46.163242467Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:31:46.211414824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:31:46.214676738Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:31:46.322338727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="727 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:31:46.325532777Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:31:46.817165643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:31:46.820673797Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:31:46.820696571Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:46.820720196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:31:46.823653563Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:31:47.083473525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:31:47.086746307Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:31:47.689244761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:31:47.692696229Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:31:47.954242081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:31:47.957472273Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:31:48.265141613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:31:48.268631587Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:31:48.268657414Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:48.268681262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:31:48.271339019Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:31:48.981145638Z caller=checkpoint.go:378 msg="writing series" size="1.3 kB"
level=info ts=2026-02-16T14:31:50.226198153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:31:50.230524345Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-02-16T14:31:50.903526853Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:52.443546539Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=info ts=2026-02-16T14:31:52.463785596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:31:52.466876572Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:31:53.730430148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:31:53.733693987Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:31:53.733721113Z caller=flush.go:240 component=ingester loop=12 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:53.733745495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:31:53.736467116Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T14:31:55.903778334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:31:55.904916184Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-02-16T14:31:56.56874523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:31:56.575177182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:31:56.575203896Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:56.575230208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:31:56.580847079Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:31:57.285736133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:31:57.288994685Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:31:57.357350048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:31:57.360329818Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:31:57.360353185Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:57.360394866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:31:57.362909033Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:31:57.435011321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:57.437898096Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:57.437921927Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:57.660633583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:31:57.663754273Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:31:58.228508053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=info ts=2026-02-16T14:31:59.034318384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="504 B" total_uncomp="983 B" avg_uncomp="491 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:31:59.039081214Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:31:59.225050926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e8fafe4bee59e2bf immediate=false num_chunks=2 total_comp="2.8 kB" avg_comp="1.4 kB" total_uncomp="12 kB" avg_uncomp="6.0 kB" full=2 labels="{stream=\"application-service\"}"
level=error ts=2026-02-16T14:31:59.228132916Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e8fafe4bee59e2bf/19c66ba458a%3A19c66ba48e7%3A9eadc5f5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"application-service\"}"
level=error ts=2026-02-16T14:31:59.228155649Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:31:59.228198026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:31:59.234289346Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T14:31:59.366185328Z caller=checkpoint.go:378 msg="writing series" size="4.6 kB"
level=info ts=2026-02-16T14:31:59.772325726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:31:59.775464214Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=debug ts=2026-02-16T14:32:00.198696479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:32:00.83467593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:32:00.837750281Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:32:00.837771523Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:00.837795082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:32:00.840170204Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:32:00.903969512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:01.188085104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:01.191089451Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:01.191111197Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:01.19113508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:01.193453867Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:32:02.684847967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:32:02.688177326Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-02-16T14:32:02.827880601Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-02-16T14:32:02.827907875Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=info ts=2026-02-16T14:32:04.092674337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:04.096035413Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:04.096060549Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:04.096087321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:04.098849498Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:32:04.199330558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:04.207739165Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:04.207766458Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:04.397103896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:32:04.39712287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:04.400463523Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:32:04.404231214Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:32:05.903600298Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:06.085880518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5b5771c832e9adb9 immediate=false num_chunks=2 total_comp="4.5 kB" avg_comp="2.2 kB" total_uncomp="13 kB" avg_uncomp="6.3 kB" full=2 labels="{stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:32:06.089536269Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5b5771c832e9adb9/19c66ba48e7%3A19c66ba8159%3A1c7274b9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-cluster-csi-drivers\"}"
level=error ts=2026-02-16T14:32:06.089566768Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:06.089593904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:32:06.094468576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3A63734c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-02-16T14:32:06.289487446Z caller=checkpoint.go:378 msg="writing series" size="131 kB"
level=info ts=2026-02-16T14:32:07.978722521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:32:07.985076645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-02-16T14:32:08.315491115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:49270"
level=info ts=2026-02-16T14:32:08.790042414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:08.793404998Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:08.793431886Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:08.79345638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:08.796136466Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:32:08.864391605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" full=2 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:32:08.867685134Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:32:08.867706533Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:09.093718436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" full=2 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:32:09.097034707Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=debug ts=2026-02-16T14:32:09.751295552Z caller=checkpoint.go:378 msg="writing series" size="671 kB"
level=info ts=2026-02-16T14:32:09.862419025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:32:09.865887243Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T14:32:09.865911153Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:09.865935788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:32:09.868598062Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:32:10.423290379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:32:10.426789217Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:32:10.904133992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:32:10.90416701Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:32:10.904176323Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:32:10.904187119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:10.933353856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:32:10.936787945Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:32:11.415405842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:32:11.418992252Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:32:12.294429552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:32:12.297411227Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:32:12.75400546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:32:12.757404364Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-02-16T14:32:13.211745922Z caller=checkpoint.go:378 msg="writing series" size="1.1 kB"
level=info ts=2026-02-16T14:32:14.368739366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ca7563d8112e730d immediate=false num_chunks=1 total_comp="3.8 kB" avg_comp="3.8 kB" total_uncomp="23 kB" avg_uncomp="23 kB" full=1 labels="{stream=\"openshift-ingress\"}"
level=error ts=2026-02-16T14:32:14.372410471Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ca7563d8112e730d/19c66ba5ca4%3A19c66ba8159%3A67acb286?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ingress\"}"
level=info ts=2026-02-16T14:32:14.431420521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:32:14.434527155Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:32:15.887050626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:32:15.890155857Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:32:15.904277782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:16.33447969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:32:16.337461281Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=debug ts=2026-02-16T14:32:16.673722779Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
level=info ts=2026-02-16T14:32:16.771110186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:32:16.779768722Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T14:32:16.843803643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:16.846818484Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:32:19.461790969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:19.465147533Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:32:19.49222032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:32:19.495299167Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=debug ts=2026-02-16T14:32:20.135057937Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=debug ts=2026-02-16T14:32:20.13508383Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-02-16T14:32:20.350604594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:20.35814154Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:32:20.903539988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:22.0293546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:22.032548614Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:32:22.370757461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:32:22.376068941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3A63734c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:32:23.017949686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:32:23.021516585Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:32:23.5972072Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=info ts=2026-02-16T14:32:24.160821066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" full=2 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:32:24.163761505Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=warn ts=2026-02-16T14:32:25.904344785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:26.038039268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:32:26.041182669Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=debug ts=2026-02-16T14:32:27.058170974Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-02-16T14:32:27.594794796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:32:27.598314282Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-02-16T14:32:28.231185085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=debug ts=2026-02-16T14:32:30.520432042Z caller=checkpoint.go:378 msg="writing series" size="510 B"
level=warn ts=2026-02-16T14:32:30.903628553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:32.430572834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:32.43404674Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:32.434067793Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-02-16T14:32:32.434089429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:32.437292375Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:32:32.896731564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:32:33.477043485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=2 total_comp="1.0 kB" avg_comp="504 B" total_uncomp="983 B" avg_uncomp="491 B" full=2 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:32:33.480507687Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:32:33.514589674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:32:33.517892654Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:32:33.827049833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:32:33.830745045Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=debug ts=2026-02-16T14:32:33.981237381Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-02-16T14:32:35.903543619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:36.275644137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-02-16T14:32:37.443512721Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-02-16T14:32:38.053413749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:32:38.057032795Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-02-16T14:32:38.357309558Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:32:38.358363281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:32:39.786982772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:32:39.790529672Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:32:39.954033504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:32:39.95735508Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:32:40.080577604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:32:40.083331804Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:32:40.903481489Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:32:40.903514824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:32:40.903523471Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:32:40.903563997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:40.905591699Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T14:32:40.905606385Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-02-16T14:32:42.306682644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=be1a6f0d81e7ecea immediate=false num_chunks=1 total_comp="670 kB" avg_comp="670 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" full=1 labels="{stream=\"build-e2e-pxgn\"}"
level=error ts=2026-02-16T14:32:42.310350816Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/be1a6f0d81e7ecea/19c66c0927b%3A19c66cd9056%3Aa31c6246?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-pxgn\"}"
level=info ts=2026-02-16T14:32:42.983444675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="489 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:32:42.991501183Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T14:32:44.366357291Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=info ts=2026-02-16T14:32:44.498130001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:32:44.504203208Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3A63734c2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:32:44.807137327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:32:44.814624333Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66cdb766%3Aacc25b47?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:32:45.464782463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=2 total_comp="10 kB" avg_comp="5.0 kB" total_uncomp="49 kB" avg_uncomp="25 kB" full=2 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:32:45.468243757Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-service-ca\"}"
level=warn ts=2026-02-16T14:32:45.903330904Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:32:46.606831525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:32:46.610113297Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-02-16T14:32:47.832284618Z caller=checkpoint.go:378 msg="writing series" size="3.5 MB"
level=debug ts=2026-02-16T14:32:47.832309444Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.5 MB" series=2
level=info ts=2026-02-16T14:32:50.111332197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:32:50.114824116Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:32:50.322415322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:32:50.325546767Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:32:50.523205258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:32:50.526167637Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:32:50.574576716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:32:50.582324289Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:32:50.904058128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:51.289360881Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=info ts=2026-02-16T14:32:52.136520435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:32:52.140185548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-02-16T14:32:54.750722264Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-02-16T14:32:55.903851043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:32:58.213086121Z caller=checkpoint.go:378 msg="writing series" size="1.1 MB"
level=debug ts=2026-02-16T14:32:58.213109858Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
ts=2026-02-16T14:32:58.234141348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:32:58.961633243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:32:58.965215104Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:32:59.981514336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:32:59.984973045Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-02-16T14:33:00.300025105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:33:00.903683899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:01.673654793Z caller=checkpoint.go:378 msg="writing series" size="919 kB"
level=info ts=2026-02-16T14:33:02.024485852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e0e9295723985086 immediate=false num_chunks=2 total_comp="90 kB" avg_comp="45 kB" total_uncomp="881 kB" avg_uncomp="441 kB" full=2 labels="{stream=\"appstudio-grafana\"}"
level=error ts=2026-02-16T14:33:02.028162807Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e0e9295723985086/19c66ba4569%3A19c66c3b8f3%3A96d0645a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"appstudio-grafana\"}"
level=info ts=2026-02-16T14:33:04.266853587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:33:04.271498835Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"