level=info ts=2026-04-22T20:31:54.399118432Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T20:31:54.399155437Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-22T20:31:54.399441354Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-22T20:31:54.400091645Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T20:31:54.400462602Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T20:31:54.400530487Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
level=info ts=2026-04-22T20:31:54.401003827Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-c7174df1
ts=2026-04-22T20:31:54.40110553Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T20:31:54.401116769Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=debug ts=2026-04-22T20:31:54.401132031Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.82 advertisePort=7946
level=info ts=2026-04-22T20:31:54.40124171Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-04-22T20:31:54.402736411Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.72:11211 count=1
level=info ts=2026-04-22T20:31:54.403670725Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-22T20:31:54.403722693Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T20:31:54.40441693Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T20:31:54.404440521Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T20:31:54.404438459Z
level=debug ts=2026-04-22T20:31:54.404495316Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-22T20:31:54.404554729Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-22T20:31:54.404577763Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-04-22T20:31:54.404585541Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-04-22T20:31:54.404584198Z
level=debug ts=2026-04-22T20:31:54.404595992Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-22T20:31:54.405858147Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T20:31:54.405939699Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.82 inf=eth0
ts=2026-04-22T20:31:54.412256652Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-04-22T20:31:54.413359686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-22T20:31:54.414943227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-22T20:31:54.415412074Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-22T20:31:54.416509213Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused"
ts=2026-04-22T20:31:54.417057486Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
level=info ts=2026-04-22T20:31:54.417064646Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=15.825151ms
level=info ts=2026-04-22T20:31:54.417087348Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-22T20:31:54.41986519Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-22T20:31:54.419893033Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T20:31:54.419893957Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T20:31:54.419914672Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-22T20:31:54.419921268Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:31:54.419931886Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-22T20:31:54.41994185Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T20:31:54.419940371Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:31:54.419952429Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-22T20:31:54.419957002Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-22T20:31:54.419957172Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-22T20:31:54.42011458Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-22T20:31:54.420122685Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-22T20:31:54.420131444Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-22T20:31:54.420135524Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-22T20:31:54.420147946Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-22T20:31:54.420218358Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-22T20:31:54.42022484Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-04-22T20:31:54.420229873Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-22T20:31:54.420276056Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-22T20:31:54.420282258Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-22T20:31:54.4202866Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-22T20:31:54.420291753Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-22T20:31:54.420296355Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-22T20:31:54.420317377Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-04-22T20:31:54.420580373Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.70:7946: dial tcp 10.130.0.70:7946: connect: connection refused"
ts=2026-04-22T20:31:54.42172858Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.72:7946"
ts=2026-04-22T20:31:54.423194438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.78:7946"
ts=2026-04-22T20:31:54.423850805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-22T20:31:54.424494098Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.83:7946: dial tcp 10.129.0.83:7946: connect: connection refused"
ts=2026-04-22T20:31:54.424605562Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
level=info ts=2026-04-22T20:31:54.424616032Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=7.521904ms
level=info ts=2026-04-22T20:31:54.634494308Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=214.184175ms errors=false
level=info ts=2026-04-22T20:31:54.634520792Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-22T20:31:54.66210638Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=241.795552ms errors=false
level=info ts=2026-04-22T20:31:54.662138618Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-22T20:31:54.662178836Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=241.868914ms
level=info ts=2026-04-22T20:31:54.662199194Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-22T20:31:54.662263514Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T20:31:54.662291106Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=13.319278014s
level=info ts=2026-04-22T20:31:54.662317511Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-04-22T20:31:54.662362035Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-22T20:31:54.662390017Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-22T20:31:54.662407082Z caller=loki.go:599 msg="Loki started" startup_time=320.363131ms
ts=2026-04-22T20:31:57.601190005Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:31:59.602080415Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
ts=2026-04-22T20:31:59.602111445Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:31:59.605014136Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
level=warn ts=2026-04-22T20:31:59.66329803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:02.470688771Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:42546"
level=warn ts=2026-04-22T20:32:04.662697014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:07.556292125Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:32:09.557561822Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
ts=2026-04-22T20:32:09.557654826Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:32:09.560501224Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
level=warn ts=2026-04-22T20:32:09.663002528Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:10.690299415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:45682"
ts=2026-04-22T20:32:12.242600609Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
ts=2026-04-22T20:32:14.243130679Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
ts=2026-04-22T20:32:14.243174265Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:32:14.243187922Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:32:14.402165765Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
level=warn ts=2026-04-22T20:32:14.663239034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T20:32:16.179523049Z caller=logging.go:137 msg="GET /ready (503) 95.477µs"
ts=2026-04-22T20:32:16.402586347Z caller=memberlist_logger.go:74 level=debug msg="Failed UDP ping: loki-ingester-0-6918225f (timeout reached)"
ts=2026-04-22T20:32:16.403374699Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:32:16.403403309Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:32:16.406123013Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
ts=2026-04-22T20:32:17.600961235Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:32:19.402746152Z caller=memberlist_logger.go:74 level=info msg="Suspect loki-ingester-0-6918225f has failed, no acks received"
ts=2026-04-22T20:32:19.601402243Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
ts=2026-04-22T20:32:19.601430331Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:32:19.60435551Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
level=warn ts=2026-04-22T20:32:19.66246511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:22.556347965Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.81:7946"
ts=2026-04-22T20:32:24.557185838Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.82:7946"
ts=2026-04-22T20:32:24.557211835Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.129.0.78:7946"
ts=2026-04-22T20:32:24.560160213Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-6918225f' from=10.130.0.72:7946"
level=info ts=2026-04-22T20:32:24.662567126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T20:32:24.662581176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T20:32:24.662608652Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-22T20:32:24.662617726Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=info ts=2026-04-22T20:32:24.662625842Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T20:32:26.177230202Z caller=logging.go:137 msg="GET /ready (503) 82.993µs"
level=warn ts=2026-04-22T20:32:29.663389381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T20:32:34.66336449Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:37.682340026Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.78:50176"
level=info ts=2026-04-22T20:32:38.56341181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="112 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T20:32:38.563445077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=info ts=2026-04-22T20:32:38.563422375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=info ts=2026-04-22T20:32:38.563409611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41f2b7a2c7d7305d immediate=false num_chunks=1 total_comp="173 kB" avg_comp="173 kB" total_uncomp="856 kB" avg_uncomp="856 kB" idle=1 labels="{stream=\"integration2-kyoz\"}"
level=info ts=2026-04-22T20:32:38.563460911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-22T20:32:38.563497912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf912d54f3915ea0 immediate=false num_chunks=1 total_comp="191 kB" avg_comp="191 kB" total_uncomp="863 kB" avg_uncomp="863 kB" full=1 labels="{stream=\"gitlab-rep-rent\"}"
level=info ts=2026-04-22T20:32:38.563511157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T20:32:38.563529024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1212783c42f85e13 immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" idle=1 labels="{stream=\"build-e2e-zoby\"}"
level=info ts=2026-04-22T20:32:38.563548528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-22T20:32:38.563557501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="517 B" avg_comp="517 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=info ts=2026-04-22T20:32:38.563576897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-22T20:32:38.563619099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=info ts=2026-04-22T20:32:38.563616888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d900cb6e7864a783 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="189 kB" avg_uncomp="189 kB" idle=1 labels="{stream=\"build-e2e-nlan\"}"
level=info ts=2026-04-22T20:32:38.563633828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=info ts=2026-04-22T20:32:38.564141248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3171f934233f729e immediate=false num_chunks=1 total_comp="781 kB" avg_comp="781 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" idle=1 labels="{stream=\"build-e2e-ybfq\"}"
level=info ts=2026-04-22T20:32:38.564344996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-22T20:32:38.564361488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-04-22T20:32:38.564379027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=info ts=2026-04-22T20:32:38.564377337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=311a0f9b84cf186d immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-mfci\"}"
level=info ts=2026-04-22T20:32:38.56440314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="177 kB" avg_uncomp="177 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-04-22T20:32:38.564412697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="622 B" avg_comp="622 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-04-22T20:32:38.564425962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.7 MB" avg_comp="1.4 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T20:32:38.564539767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=info ts=2026-04-22T20:32:38.564551537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T20:32:38.564595859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T20:32:38.564610081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9cda38d9fe9ce247 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-vqlw\"}"
level=error ts=2026-04-22T20:32:38.56936873Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db68b372f%3A19db69d9d32%3A6b9ff1e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T20:32:38.569370682Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db68b41ca%3A19db6d81055%3Ad48f27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-22T20:32:38.569401777Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db68b34c1%3A19db68c339d%3A3a0c34d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-22T20:32:38.569439104Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db68b359c%3A19db6b61c88%3Ac7d428a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T20:32:38.569430941Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41f2b7a2c7d7305d/19db6a385c8%3A19db6a764b0%3A1123aaba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-kyoz\"}"
level=error ts=2026-04-22T20:32:38.569442887Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db68c1206%3A19db68c5d05%3Aa9896337?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-22T20:32:38.569456415Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db695cb7c%3A19db69750c9%3Ae1415c85?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-22T20:32:38.569448163Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db68c11d9%3A19db69f0571%3Afbad3202?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-22T20:32:38.569449743Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db68b432b%3A19db69ab54e%3Af1d26403?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-22T20:32:38.569452519Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/311a0f9b84cf186d/19db69638dd%3A19db6ab3fad%3Af49640ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mfci\"}"
level=error ts=2026-04-22T20:32:38.569472392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db68b34ea%3A19db6c974b9%3A9a3357f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-22T20:32:38.569498955Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db68c1206%3A19db68c1207%3A4632d35a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-22T20:32:38.569499692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db68c339c%3A19db68c339d%3A2bc02e2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-22T20:32:38.569488323Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db68c11d9%3A19db6a9b4b5%3Aa91d8b6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T20:32:38.569503545Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db68c3100%3A19db68c3650%3A26740bef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-22T20:32:38.569517267Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db68c30ab%3A19db68c339d%3Ab3de6dde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-22T20:32:38.569521373Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db68c2db6%3A19db68c339d%3A41c310b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-22T20:32:38.569549461Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d900cb6e7864a783/19db6959378%3A19db6a85337%3A1ca949c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlan\"}"
level=error ts=2026-04-22T20:32:38.569533616Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db68c2db6%3A19db68c847e%3A165dfaaf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-22T20:32:38.569537539Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf912d54f3915ea0/19db695c258%3A19db69e3a51%3A10fcea6f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-rent\"}"
level=error ts=2026-04-22T20:32:38.571660183Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db68b3e56%3A19db68c290f%3A45a3c035?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T20:32:38.571660774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db68b47c7%3A19db695a63c%3A88d6a63d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T20:32:38.571673591Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cda38d9fe9ce247/19db695a20e%3A19db6a79ac1%3A455d5c92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-vqlw\"}"
level=error ts=2026-04-22T20:32:38.573402359Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1212783c42f85e13/19db696a100%3A19db6ab5c8f%3A4330330b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zoby\"}"
level=error ts=2026-04-22T20:32:38.580188576Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3171f934233f729e/19db696bc58%3A19db6ab54bf%3A27b362fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ybfq\"}"
level=warn ts=2026-04-22T20:32:39.663129263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:40.693892014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.72:47150"
level=warn ts=2026-04-22T20:32:44.662634487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T20:32:45.667253302Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-c7cqh-0f21ea0b 10.129.0.78:7946"
level=info ts=2026-04-22T20:32:48.828821679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="741 kB" avg_comp="741 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-22T20:32:48.833814935Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19db68b34ea%3A19db6c974b9%3A9a3357f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-22T20:32:49.63697561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="535 B" avg_comp="535 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-22T20:32:49.639843341Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19db68c339c%3A19db68c339d%3A2bc02e2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-22T20:32:49.662948309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T20:32:49.931757026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T20:32:49.935313066Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db68b432b%3A19db69ab54e%3Af1d26403?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-22T20:32:50.249481242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.0 MB" avg_comp="2.0 MB" total_uncomp="27 MB" avg_uncomp="13 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T20:32:50.254537554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db68b372f%3A19db69d9d32%3A6b9ff1e1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T20:32:50.307782985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.2 MB" avg_uncomp="4.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-22T20:32:50.3109079Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19db68b41ca%3A19db6d81055%3Ad48f27fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-22T20:32:50.511744397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-22T20:32:50.514722145Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19db68c2db6%3A19db68c339d%3A41c310b3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-22T20:32:51.603032091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-22T20:32:51.605981805Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19db68c11d9%3A19db69f0571%3Afbad3202?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-22T20:32:52.071123004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="177 kB" avg_uncomp="177 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-22T20:32:52.074676354Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19db68c2db6%3A19db68c847e%3A165dfaaf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-22T20:32:52.089737966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=41f2b7a2c7d7305d immediate=false num_chunks=1 total_comp="192 kB" avg_comp="192 kB" total_uncomp="856 kB" avg_uncomp="856 kB" full=1 labels="{stream=\"integration2-kyoz\"}"
level=error ts=2026-04-22T20:32:52.092723642Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/41f2b7a2c7d7305d/19db6a385c8%3A19db6a764b0%3A1123aaba?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration2-kyoz\"}"
level=info ts=2026-04-22T20:32:52.299384827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-22T20:32:52.302226059Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19db68c1206%3A19db68c5d05%3Aa9896337?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-22T20:32:52.465071775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-22T20:32:52.468044599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19db68b34c1%3A19db68c339d%3A3a0c34d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-22T20:32:53.590169557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="640 B" avg_comp="640 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-22T20:32:53.593044487Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19db68c1206%3A19db68c1207%3A4632d35a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-22T20:32:54.164125275Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T20:32:54.189131579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d900cb6e7864a783 immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="189 kB" avg_uncomp="189 kB" full=1 labels="{stream=\"build-e2e-nlan\"}"
level=error ts=2026-04-22T20:32:54.191995963Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d900cb6e7864a783/19db6959378%3A19db6a85337%3A1ca949c0?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlan\"}"
level=info ts=2026-04-22T20:32:54.21835614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-22T20:32:54.221165339Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19db68c3100%3A19db68c3650%3A26740bef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-22T20:32:54.40397823Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T20:32:54.403998092Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.72:11211 count=1
level=info ts=2026-04-22T20:32:54.535064362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.5 MB" avg_comp="1.5 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-22T20:32:54.539795556Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19db68b359c%3A19db6b61c88%3Ac7d428a8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-22T20:32:54.650140889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=cf912d54f3915ea0 immediate=false num_chunks=1 total_comp="191 kB" avg_comp="191 kB" total_uncomp="863 kB" avg_uncomp="863 kB" full=1 labels="{stream=\"gitlab-rep-rent\"}"
level=error ts=2026-04-22T20:32:54.653497328Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/cf912d54f3915ea0/19db695c258%3A19db69e3a51%3A10fcea6f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-rent\"}"
level=info ts=2026-04-22T20:32:54.662540711Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T20:32:54.662551764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T20:32:54.662562568Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T20:32:54.662571686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T20:32:54.704879005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.4 kB" avg_comp="6.4 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-22T20:32:54.707873089Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19db68c30ab%3A19db68c339d%3Ab3de6dde?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-22T20:32:55.043887338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3171f934233f729e immediate=false num_chunks=1 total_comp="794 kB" avg_comp="794 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" full=1 labels="{stream=\"build-e2e-ybfq\"}"
level=error ts=2026-04-22T20:32:55.049156835Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3171f934233f729e/19db696bc58%3A19db6ab54bf%3A27b362fc?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ybfq\"}"
level=info ts=2026-04-22T20:32:55.058769126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1212783c42f85e13 immediate=false num_chunks=1 total_comp="388 kB" avg_comp="388 kB" total_uncomp="2.5 MB" avg_uncomp="2.5 MB" full=1 labels="{stream=\"build-e2e-zoby\"}"
level=error ts=2026-04-22T20:32:55.061841243Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1212783c42f85e13/19db696a100%3A19db6ab5c8f%3A4330330b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zoby\"}"
level=info ts=2026-04-22T20:32:55.784134355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9cda38d9fe9ce247 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-vqlw\"}"
level=error ts=2026-04-22T20:32:55.790846326Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9cda38d9fe9ce247/19db695a20e%3A19db6a79ac1%3A455d5c92?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-vqlw\"}"
level=info ts=2026-04-22T20:32:55.957970641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="2.7 MB" avg_comp="1.4 MB" total_uncomp="18 MB" avg_uncomp="8.8 MB" synced=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-22T20:32:55.963324223Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19db68c11d9%3A19db6a9b4b5%3Aa91d8b6b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-22T20:32:56.133234924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=311a0f9b84cf186d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-mfci\"}"
level=error ts=2026-04-22T20:32:56.136012824Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/311a0f9b84cf186d/19db69638dd%3A19db6ab3fad%3Af49640ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-mfci\"}"
level=info ts=2026-04-22T20:32:56.17895968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="112 MB" avg_uncomp="22 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T20:32:56.184495082Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db68b3e56%3A19db68c290f%3A45a3c035?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T20:32:57.480309666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="57 MB" avg_comp="2.8 MB" total_uncomp="464 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T20:32:57.487165647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db68b47c7%3A19db695a63c%3A88d6a63d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T20:32:58.001829357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-22T20:32:58.004659458Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19db695cb7c%3A19db69750c9%3Ae1415c85?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-22T20:32:59.662813089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"