level=info ts=2026-04-22T20:46:00.497819346Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T20:46:00.497847273Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-22T20:46:00.497863948Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-22T20:46:00.498777675Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T20:46:00.499133535Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T20:46:00.499208386Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.69 inf=eth0
level=info ts=2026-04-22T20:46:00.499687099Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-5b188008
ts=2026-04-22T20:46:00.499755664Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T20:46:00.49976762Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=debug ts=2026-04-22T20:46:00.499785696Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.69 advertisePort=7946
level=info ts=2026-04-22T20:46:00.499898561Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-22T20:46:00.501402985Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.99:11211 count=1
level=info ts=2026-04-22T20:46:00.50224358Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-22T20:46:00.50229344Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T20:46:00.50342679Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T20:46:00.50344452Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T20:46:00.503443373Z
level=debug ts=2026-04-22T20:46:00.50345872Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-22T20:46:00.503472709Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-22T20:46:00.503485706Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-04-22T20:46:00.503490086Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-22T20:46:00.503489584Z
level=debug ts=2026-04-22T20:46:00.503493946Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-22T20:46:00.504587361Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T20:46:00.504656536Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.69 inf=eth0
ts=2026-04-22T20:46:00.50694239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.71:7946"
ts=2026-04-22T20:46:00.509333185Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.70:7946: dial tcp 10.129.0.70:7946: connect: connection refused"
ts=2026-04-22T20:46:00.510279703Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.96:7946"
ts=2026-04-22T20:46:00.513317345Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.97:7946: dial tcp 10.130.0.97:7946: connect: connection refused"
ts=2026-04-22T20:46:00.514177585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.93:7946"
ts=2026-04-22T20:46:00.516827582Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.98:7946: dial tcp 10.130.0.98:7946: connect: connection refused"
level=info ts=2026-04-22T20:46:00.516836948Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=16.940699ms
level=info ts=2026-04-22T20:46:00.516853122Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-22T20:46:00.518275699Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:46:00.518359931Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-22T20:46:00.51837074Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T20:46:00.518363335Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-22T20:46:00.518387073Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-22T20:46:00.518362018Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:46:00.518680531Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=info ts=2026-04-22T20:46:00.518704956Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T20:46:00.518718316Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-22T20:46:00.518723506Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=debug ts=2026-04-22T20:46:00.518722432Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-04-22T20:46:00.518728032Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-22T20:46:00.518729963Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:46:00.518734745Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-22T20:46:00.518741766Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-22T20:46:00.518746199Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-22T20:46:00.518749951Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-22T20:46:00.518839318Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-22T20:46:00.51886356Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:46:00.518868444Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-22T20:46:00.518871228Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-22T20:46:00.51887409Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-22T20:46:00.518876626Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-22T20:46:00.51888037Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-22T20:46:00.518891465Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-04-22T20:46:00.519525299Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.71:7946"
ts=2026-04-22T20:46:00.520058968Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.70:7946: dial tcp 10.129.0.70:7946: connect: connection refused"
ts=2026-04-22T20:46:00.520940617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.96:7946"
ts=2026-04-22T20:46:00.522823259Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.97:7946: dial tcp 10.130.0.97:7946: connect: connection refused"
ts=2026-04-22T20:46:00.52366875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.93:7946"
ts=2026-04-22T20:46:00.525637003Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.98:7946: dial tcp 10.130.0.98:7946: connect: connection refused"
level=info ts=2026-04-22T20:46:00.525649956Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=8.79245ms
level=info ts=2026-04-22T20:46:00.733525708Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=214.633799ms errors=false
level=info ts=2026-04-22T20:46:00.733556994Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-22T20:46:00.773411702Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=254.520216ms errors=false
level=info ts=2026-04-22T20:46:00.773436543Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-22T20:46:00.773465013Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=254.574178ms
level=info ts=2026-04-22T20:46:00.773484708Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-22T20:46:00.773523573Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T20:46:00.773529036Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=9.141177276s
level=info ts=2026-04-22T20:46:00.773562657Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-04-22T20:46:00.773599853Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-22T20:46:00.773626104Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-22T20:46:00.773641546Z caller=loki.go:599 msg="Loki started" startup_time=333.007175ms
level=warn ts=2026-04-22T20:46:05.773745472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T20:46:10.774259154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:15.367606578Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
ts=2026-04-22T20:46:15.500601902Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
level=warn ts=2026-04-22T20:46:15.774031284Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T20:46:16.972165391Z caller=logging.go:137 msg="GET /ready (503) 99.933µs"
ts=2026-04-22T20:46:17.367978401Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
ts=2026-04-22T20:46:17.36802972Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:17.36804424Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:17.500523518Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-01315b5c (timeout reached)"
ts=2026-04-22T20:46:17.5012963Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:17.503456406Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:17.503483538Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
ts=2026-04-22T20:46:20.50134327Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-01315b5c has failed, no acks received"
level=warn ts=2026-04-22T20:46:20.774650526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:25.464369673Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:25.50076904Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
level=warn ts=2026-04-22T20:46:25.773965711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T20:46:26.970919153Z caller=logging.go:137 msg="GET /ready (503) 77.669µs"
ts=2026-04-22T20:46:27.46507736Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
ts=2026-04-22T20:46:27.467283563Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
ts=2026-04-22T20:46:27.46732351Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:27.50160969Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-01315b5c (timeout reached)"
ts=2026-04-22T20:46:27.50223559Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:27.504331889Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
ts=2026-04-22T20:46:27.504356384Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:30.362953863Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:30.501170478Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-01315b5c has failed, no acks received"
level=warn ts=2026-04-22T20:46:30.774413032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T20:46:30.774408389Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-22T20:46:30.774438357Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-22T20:46:30.774453027Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=info ts=2026-04-22T20:46:30.774472119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-22T20:46:32.363656303Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
ts=2026-04-22T20:46:32.363690129Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:32.363699874Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
level=warn ts=2026-04-22T20:46:35.774347243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:40.464136376Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.71:7946"
ts=2026-04-22T20:46:40.50195107Z caller=memberlist_logger.go:74 level=info msg="Marking loki-ingester-0-01315b5c as failed, suspect timeout reached (2 peer confirmations)"
level=warn ts=2026-04-22T20:46:40.774151887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:42.464820838Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.129.0.69:7946"
ts=2026-04-22T20:46:42.467087074Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.93:7946"
ts=2026-04-22T20:46:42.46725199Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-01315b5c' from=10.130.0.96:7946"
level=info ts=2026-04-22T20:46:43.899581959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a8d437ba6ea3921c immediate=false num_chunks=1 total_comp="1.8 kB" avg_comp="1.8 kB" total_uncomp="186 kB" avg_uncomp="186 kB" idle=1 labels="{stream=\"build-e2e-ytad\"}"
level=info ts=2026-04-22T20:46:43.899608483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=info ts=2026-04-22T20:46:43.899606603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-22T20:46:43.899626591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-22T20:46:43.899638068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="131 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T20:46:43.899646937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a789278ee527825 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"build-e2e-rpfm\"}"
level=info ts=2026-04-22T20:46:43.899680208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="754 kB" avg_comp="754 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=info ts=2026-04-22T20:46:43.899685215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5f5e93cd14ffc117 immediate=false num_chunks=1 total_comp="191 kB" avg_comp="191 kB" total_uncomp="864 kB" avg_uncomp="864 kB" full=1 labels="{stream=\"integration2-dbah\"}"
level=info ts=2026-04-22T20:46:43.899695804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="527 kB" avg_comp="527 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=info ts=2026-04-22T20:46:43.899700488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=info ts=2026-04-22T20:46:43.899732318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="476 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T20:46:43.899773481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=564d0d9425b3468d immediate=false num_chunks=1 total_comp="626 B" avg_comp="626 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-dboc\"}"
level=info ts=2026-04-22T20:46:43.899776262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-22T20:46:43.899799534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc4625e8c2b0c52b immediate=false num_chunks=1 total_comp="236 kB" avg_comp="236 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" idle=1 labels="{stream=\"gitlab-rep-jeut\"}"
level=info ts=2026-04-22T20:46:43.899741138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="1.9 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T20:46:43.899854066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.3 MB" total_uncomp="20 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-22T20:46:43.899978654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.7 MB" avg_comp="1.4 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T20:46:43.900107557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="148 kB" avg_uncomp="148 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-04-22T20:46:43.90025608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-22T20:46:43.900262279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-04-22T20:46:43.900267416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=info ts=2026-04-22T20:46:43.900346471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="412 kB" avg_comp="412 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-22T20:46:43.900348345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="486 kB" avg_comp="486 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=debug ts=2026-04-22T20:46:43.900523597Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T20:46:43.900589017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a784ad15667567bd immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-qhul\"}"
level=error ts=2026-04-22T20:46:43.905207487Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db69154aa%3A19db691f226%3A30597e13?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-22T20:46:43.905227921Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db691df86%3A19db691df87%3A2c34de9f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-22T20:46:43.905247838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db691d908%3A19db691eaa7%3A6016da1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-22T20:46:43.905264281Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db691bed9%3A19db6ac30f3%3Af90b578a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-22T20:46:43.905259571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db6915346%3A19db6d4bbdb%3Adad7a626?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-22T20:46:43.905276699Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db691510e%3A19db69dc57e%3A522ff91e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T20:46:43.905285848Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db691beaa%3A19db6b60de9%3A23145e6a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T20:46:43.905283471Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db6916992%3A19db6a9816e%3A38ebf8fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T20:46:43.905290956Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5f5e93cd14ffc117/19db6af808d%3A19db6b383ee%3Ab66d28ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-dbah\"}"
level=error ts=2026-04-22T20:46:43.90530031Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db6a29014%3A19db6a3fb5d%3A4d511726?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-22T20:46:43.905334988Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db691bed9%3A19db6920d85%3A25cba3a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-22T20:46:43.905346851Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19db691bed9%3A19db691eaa7%3Ac2e9b375?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-22T20:46:43.905339198Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/564d0d9425b3468d/19db6a245dd%3A19db6a26cee%3Ad7144596?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-dboc\"}"
level=error ts=2026-04-22T20:46:43.905340435Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db6915490%3A19db6c9712e%3A83b67525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-22T20:46:43.905297645Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a8d437ba6ea3921c/19db6a293fd%3A19db6b6c007%3A936d1305?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ytad\"}"
level=error ts=2026-04-22T20:46:43.905353146Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a784ad15667567bd/19db6a1a1cc%3A19db6a8e145%3Ac3696f93?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qhul\"}"
level=error ts=2026-04-22T20:46:43.905353982Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db691beaa%3A19db6d607e2%3Ad14f542c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-22T20:46:43.905360803Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db691beaa%3A19db6c9bf5b%3A2599e115?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-22T20:46:43.90536668Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db691533d%3A19db6e5424d%3Ad722d6c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-22T20:46:43.905366024Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db691686f%3A19db691dff7%3A21fd938d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-22T20:46:43.907238393Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db69169d2%3A19db6a28cdf%3Ad0a8c040?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T20:46:43.907288534Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a789278ee527825/19db6a2b33c%3A19db6c6832e%3Abdd8f80d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rpfm\"}"
level=error ts=2026-04-22T20:46:43.907338693Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc4625e8c2b0c52b/19db6a31cb4%3A19db6acadf3%3A1cee00d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-jeut\"}"
level=error ts=2026-04-22T20:46:43.908798336Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db6915803%3A19db691df87%3A1fad7ac3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T20:46:45.774103231Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T20:46:50.774388167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:53.256479693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-1f7c9756 10.129.0.71:7946"
level=info ts=2026-04-22T20:46:54.671989081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="439 kB" avg_comp="439 kB" total_uncomp="4.1 MB" avg_uncomp="4.1 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-22T20:46:54.676942214Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db691beaa%3A19db6d607e2%3Ad14f542c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-22T20:46:55.330912535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-22T20:46:55.334389373Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db691d908%3A19db691eaa7%3A6016da1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-22T20:46:55.46359319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.7 MB" avg_comp="1.4 MB" total_uncomp="18 MB" avg_uncomp="8.9 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T20:46:55.467420687Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db6916992%3A19db6a9816e%3A38ebf8fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T20:46:55.630846293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a8d437ba6ea3921c immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="186 kB" avg_uncomp="186 kB" full=1 labels="{stream=\"build-e2e-ytad\"}"
level=error ts=2026-04-22T20:46:55.634056775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a8d437ba6ea3921c/19db6a293fd%3A19db6b6c007%3A936d1305?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ytad\"}"
level=info ts=2026-04-22T20:46:55.760099402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-22T20:46:55.763264763Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db691bed9%3A19db6920d85%3A25cba3a6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-22T20:46:55.774229878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:46:55.993370031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.71:60794"
level=info ts=2026-04-22T20:46:56.793199273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="754 kB" avg_comp="754 kB" total_uncomp="3.9 MB" avg_uncomp="3.9 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-22T20:46:56.796986621Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db6915490%3A19db6c9712e%3A83b67525?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-22T20:46:57.292341669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=564d0d9425b3468d immediate=false num_chunks=1 total_comp="644 B" avg_comp="644 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-dboc\"}"
level=error ts=2026-04-22T20:46:57.295704876Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/564d0d9425b3468d/19db6a245dd%3A19db6a26cee%3Ad7144596?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-dboc\"}"
level=info ts=2026-04-22T20:46:57.551694991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="131 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T20:46:57.55798707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db6915803%3A19db691df87%3A1fad7ac3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T20:46:57.589066582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5f5e93cd14ffc117 immediate=false num_chunks=1 total_comp="191 kB" avg_comp="191 kB" total_uncomp="864 kB" avg_uncomp="864 kB" full=1 labels="{stream=\"integration2-dbah\"}"
level=error ts=2026-04-22T20:46:57.59219957Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5f5e93cd14ffc117/19db6af808d%3A19db6b383ee%3Ab66d28ac?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-dbah\"}"
level=info ts=2026-04-22T20:46:57.698557428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9a789278ee527825 immediate=false num_chunks=1 total_comp="2.1 MB" avg_comp="2.1 MB" total_uncomp="15 MB" avg_uncomp="15 MB" synced=1 labels="{stream=\"build-e2e-rpfm\"}"
level=error ts=2026-04-22T20:46:57.704216596Z caller=flush.go:262 component=ingester loop=5 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9a789278ee527825/19db6a2b33c%3A19db6c6832e%3Abdd8f80d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-rpfm\"}"
level=info ts=2026-04-22T20:46:57.875518147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc4625e8c2b0c52b immediate=false num_chunks=1 total_comp="255 kB" avg_comp="255 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-jeut\"}"
level=error ts=2026-04-22T20:46:57.878984771Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc4625e8c2b0c52b/19db6a31cb4%3A19db6acadf3%3A1cee00d6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-jeut\"}"
ts=2026-04-22T20:46:57.923532967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.96:58826"
level=info ts=2026-04-22T20:46:58.483275857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="476 MB" avg_uncomp="24 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T20:46:58.489893595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db69169d2%3A19db6a28cdf%3Ad0a8c040?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T20:46:58.557246709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="3.9 MB" avg_comp="1.9 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T20:46:58.561437269Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db691510e%3A19db69dc57e%3A522ff91e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T20:46:58.750047418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-22T20:46:58.753374541Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db691bed9%3A19db6ac30f3%3Af90b578a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-22T20:46:59.734511994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="412 kB" avg_comp="412 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-22T20:46:59.73838701Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19db691beaa%3A19db6c9bf5b%3A2599e115?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-22T20:46:59.853577989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-22T20:46:59.856855953Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db691686f%3A19db691dff7%3A21fd938d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-22T20:47:00.502356648Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T20:47:00.502586367Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.99:11211 count=1
level=info ts=2026-04-22T20:47:00.74528175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="486 kB" avg_comp="486 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-22T20:47:00.748793545Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19db691533d%3A19db6e5424d%3Ad722d6c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-22T20:47:00.774599234Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T20:47:00.774623403Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T20:47:00.774633532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T20:47:00.774635894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T20:47:01.200740245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=a784ad15667567bd immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-qhul\"}"
level=error ts=2026-04-22T20:47:01.203826846Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/a784ad15667567bd/19db6a1a1cc%3A19db6a8e145%3Ac3696f93?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-qhul\"}"
level=info ts=2026-04-22T20:47:01.889522736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="527 kB" avg_comp="527 kB" total_uncomp="5.3 MB" avg_uncomp="5.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T20:47:01.89326465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db6915346%3A19db6d4bbdb%3Adad7a626?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-22T20:47:02.009839753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-22T20:47:02.012950762Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db6a29014%3A19db6a3fb5d%3A4d511726?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-22T20:47:02.287167479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="29 kB" avg_uncomp="29 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-22T20:47:02.290244704Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db691df86%3A19db691df87%3A2c34de9f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-22T20:47:02.332534325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=2 total_comp="2.5 MB" avg_comp="1.3 MB" total_uncomp="20 MB" avg_uncomp="10 MB" synced=2 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T20:47:02.336389092Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db691beaa%3A19db6b60de9%3A23145e6a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-22T20:47:02.436905633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T20:47:02.755899596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="31 kB" avg_comp="31 kB" total_uncomp="148 kB" avg_uncomp="148 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-22T20:47:02.758920003Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db69154aa%3A19db691f226%3A30597e13?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-22T20:47:03.88495734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-22T20:47:03.888077139Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19db691bed9%3A19db691eaa7%3Ac2e9b375?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-22T20:47:05.773739023Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T20:47:10.774208674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T20:47:15.774110413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"