level=info ts=2026-02-16T14:33:33.310426566Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-02-16T14:33:33.310473679Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-02-16T14:33:33.310498445Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-02-16T14:33:33.311724435Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-02-16T14:33:33.312212256Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T14:33:33.312301855Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
level=info ts=2026-02-16T14:33:33.31297558Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-81e5c3b7
ts=2026-02-16T14:33:33.313058154Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-02-16T14:33:33.31307236Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=debug ts=2026-02-16T14:33:33.313092445Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.130.0.68 advertisePort=7946
level=info ts=2026-02-16T14:33:33.313211059Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=debug ts=2026-02-16T14:33:33.316098385Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:33:33.317428976Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-02-16T14:33:33.317471187Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-02-16T14:33:33.31840076Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-02-16T14:33:33.318427373Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-02-16T14:33:33.318425347Z
level=debug ts=2026-02-16T14:33:33.318452611Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-02-16T14:33:33.318676865Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-02-16T14:33:33.318715763Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=1
level=debug ts=2026-02-16T14:33:33.3187229Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=1 ts=2026-02-16T14:33:33.318722072Z
level=debug ts=2026-02-16T14:33:33.318728277Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-02-16T14:33:33.319836311Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-02-16T14:33:33.319920921Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.130.0.68 inf=eth0
ts=2026-02-16T14:33:33.322123456Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T14:33:33.324304427Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-02-16T14:33:33.325933174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-02-16T14:33:33.32980218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-02-16T14:33:33.332047846Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
level=debug ts=2026-02-16T14:33:33.334100649Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-02-16T14:33:33.334168975Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-02-16T14:33:33.334164491Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-02-16T14:33:33.334193312Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-02-16T14:33:33.334189504Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-02-16T14:33:33.334190665Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-02-16T14:33:33.334204005Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-02-16T14:33:33.334659157Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-02-16T14:33:33.334669142Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-02-16T14:33:33.334751557Z caller=module_service.go:82 msg=starting module=memberlist-kv
ts=2026-02-16T14:33:33.33484653Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
level=info ts=2026-02-16T14:33:33.334861867Z caller=memberlist_client.go:648 msg="memberlist fast-join finished" joined_nodes=3 elapsed_time=21.65296ms
level=info ts=2026-02-16T14:33:33.334916645Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-02-16T14:33:33.334912862Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-02-16T14:33:33.334951031Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-02-16T14:33:33.334958476Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-02-16T14:33:33.334963855Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-02-16T14:33:33.334966805Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-02-16T14:33:33.334974749Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-02-16T14:33:33.334981987Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-02-16T14:33:33.334963479Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-02-16T14:33:33.335246437Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-02-16T14:33:33.335255797Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-02-16T14:33:33.335260236Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-02-16T14:33:33.335263359Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-02-16T14:33:33.335271286Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-02-16T14:33:33.335327991Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-02-16T14:33:33.335350614Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
ts=2026-02-16T14:33:33.338263485Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.67:7946: dial tcp 10.130.0.67:7946: connect: connection refused"
ts=2026-02-16T14:33:33.339418743Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.65:7946: dial tcp 10.129.0.65:7946: connect: connection refused"
ts=2026-02-16T14:33:33.340519854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.66:7946"
ts=2026-02-16T14:33:33.342445472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.72:7946"
ts=2026-02-16T14:33:33.343985402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-02-16T14:33:33.345352924Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
level=info ts=2026-02-16T14:33:33.345400542Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=10.444242ms
level=info ts=2026-02-16T14:33:33.417031142Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=81.680845ms errors=false
level=info ts=2026-02-16T14:33:33.417057133Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
ts=2026-02-16T14:33:33.923493006Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-ee30c144' from=10.128.0.73:7946"
ts=2026-02-16T14:33:35.92488422Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-ee30c144' from=10.128.0.72:7946"
ts=2026-02-16T14:33:35.925125041Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-ee30c144' from=10.130.0.68:7946"
ts=2026-02-16T14:33:35.92671284Z caller=memberlist_logger.go:74 level=warn msg="Got ping for unexpected node 'loki-ingester-0-ee30c144' from=10.129.0.66:7946"
level=info ts=2026-02-16T14:33:36.663576727Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=3.328225172s errors=false
level=info ts=2026-02-16T14:33:36.66361641Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-02-16T14:33:36.663664885Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=3.328314636s
level=info ts=2026-02-16T14:33:36.663759332Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-02-16T14:33:36.663751372Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-02-16T14:33:36.663804761Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=14.855224371s
level=info ts=2026-02-16T14:33:36.663820409Z caller=lifecycler.go:740 component=ingester msg="existing instance found in ring" state=ACTIVE tokens=128 ring=ingester readOnly=false readOnlyStateUpdate=0001-01-01T00:00:00Z
level=debug ts=2026-02-16T14:33:36.663864301Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-02-16T14:33:36.663918217Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 1m0s with no heartbeat"
level=info ts=2026-02-16T14:33:36.66394299Z caller=loki.go:599 msg="Loki started" startup_time=3.413954713s
level=warn ts=2026-02-16T14:33:41.664960008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:33:46.664497948Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:33:51.66476325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:33:56.664686987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:33:57.131799649Z caller=logging.go:137 msg="GET /ready (503) 109.987µs"
level=warn ts=2026-02-16T14:34:01.6642459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:34:06.664920056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:06.664920226Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-02-16T14:34:06.664956871Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-02-16T14:34:06.664969612Z caller=recalculate_owned_streams.go:82 msg="fixed limit has been updated" tenant=kubearchive old=2147483647 new=10000
level=info ts=2026-02-16T14:34:06.664980297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-02-16T14:34:07.126832587Z caller=logging.go:137 msg="GET /ready (503) 74.321µs"
ts=2026-02-16T14:34:08.321751986Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:45772"
ts=2026-02-16T14:34:10.592819364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=warn ts=2026-02-16T14:34:11.664761468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-02-16T14:34:16.294190186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:36228"
level=warn ts=2026-02-16T14:34:16.665015325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:19.417471769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="621 B" avg_comp="621 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=info ts=2026-02-16T14:34:19.417508687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b51c839d6227bed immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"build-e2e-edsn\"}"
level=info ts=2026-02-16T14:34:19.417486429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:34:19.417471693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:34:19.417537182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="642 B" avg_comp="642 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:34:19.417542323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:34:19.417557589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-02-16T14:34:19.417585918Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:34:19.417585838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:34:19.417592324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=info ts=2026-02-16T14:34:19.417601209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="103 kB" avg_comp="103 kB" total_uncomp="668 kB" avg_uncomp="668 kB" idle=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:34:19.417634995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="931 B" avg_comp="931 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:34:19.424967159Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:19.42497096Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:34:19.424971231Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-02-16T14:34:19.425007476Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:34:19.425024704Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66ba48e6%3A19c66ba48e7%3A9da742c5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-02-16T14:34:19.425025131Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:34:19.425039667Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:34:19.425043071Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:34:19.426617492Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66ccda21%3Abb35f913?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:34:19.429015121Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b51c839d6227bed/19c66c055c2%3A19c66cd71c4%3A8e0d5b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-edsn\"}"
level=error ts=2026-02-16T14:34:19.457657111Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:34:19.718438692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:34:19.721411224Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:34:20.018388137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:34:20.021680898Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:34:20.318448393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="727 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" idle=1 full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:34:20.321537916Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:34:20.618404358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.1 kB" avg_comp="9.1 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:34:20.621252726Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:34:20.917913722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="319 B" avg_comp="319 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:34:20.920929402Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:34:21.217699592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:34:21.225002886Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:34:21.518257382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:34:21.52139999Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=warn ts=2026-02-16T14:34:21.664701895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:21.8183485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:34:21.823524276Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3Aa0602f12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:34:22.117932138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:34:22.12459011Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:34:22.4184491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="438 B" avg_comp="438 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:34:22.421197329Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:34:22.719102024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:34:22.722111031Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:34:23.018673667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.0 kB" avg_comp="1.0 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:34:23.021501373Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:34:23.317838543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="490 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:34:23.325265048Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T14:34:23.618631244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:34:23.621863943Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:34:23.91785421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="519 B" avg_comp="519 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:34:23.92098242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-02-16T14:34:26.664439523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:30.906255768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:34:30.909229594Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:34:31.026670889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:34:31.027958905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:34:31.02973956Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:34:31.029751906Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:34:31.143162459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T14:34:31.146032798Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66ba48e6%3A19c66ba48e7%3A9da742c5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T14:34:31.547357843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:34:31.550652787Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-02-16T14:34:31.664954531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:33.177844755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:34:33.317728361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:34:33.31774272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:34:33.352346062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:34:33.358881761Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:34:33.488051725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:34:33.495725285Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:34:33.798727158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:34:33.802145292Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:34:33.901976742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:34:33.905288526Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:34:34.302491532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:34:34.306148678Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:34:34.484912129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:34:34.488346266Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:34:35.203959322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:34:35.207359416Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-02-16T14:34:35.843119885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40992"
level=info ts=2026-02-16T14:34:36.211773965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:36.215140266Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:34:36.256188962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:34:36.259482671Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:34:36.535446856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:34:36.542723687Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66ccda21%3Abb35f913?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:34:36.664979256Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:34:36.665013664Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:34:36.665022093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:34:36.664988569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:36.902220626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:34:36.905546687Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:34:37.434420089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b51c839d6227bed immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"build-e2e-edsn\"}"
level=error ts=2026-02-16T14:34:37.442115635Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b51c839d6227bed/19c66c055c2%3A19c66cd71c4%3A8e0d5b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-edsn\"}"
level=info ts=2026-02-16T14:34:37.632658696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:34:37.635652792Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:34:37.741074069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:34:37.743954037Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:34:39.038120823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:34:39.041060838Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:34:39.079126629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:34:39.082494256Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:34:39.082544169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:34:39.084949987Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:34:39.865673939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:34:39.871808504Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3Aa0602f12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:34:40.033410843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:34:40.036466659Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-02-16T14:34:40.594945314Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:34:40.998322653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:34:41.001704602Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:34:41.664113769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:43.180727061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="490 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:34:43.188528541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-02-16T14:34:43.521207012Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-02-16T14:34:46.29771592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:56240"
level=warn ts=2026-02-16T14:34:46.664168518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:34:51.664362357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:34:53.955926519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-02-16T14:34:54.813874747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:34:54.816752319Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:34:54.820875863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:34:54.82343004Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-02-16T14:34:55.059022154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:34:55.062092058Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:34:55.461296267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:34:55.46446573Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:34:55.636999902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:34:55.644012306Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-02-16T14:34:56.664947217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:34:57.20270948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:34:57.205680812Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:34:57.58191598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:34:57.584948081Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-02-16T14:34:57.684183305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:34:57.690943745Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66ccda21%3Abb35f913?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:34:57.844493462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:34:57.847485365Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:35:00.152430658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:35:00.155311209Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:35:00.326012093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T14:35:00.329181231Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66ba48e6%3A19c66ba48e7%3A9da742c5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T14:35:00.813576047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:35:00.816652729Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-02-16T14:35:01.663984072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:06.664674695Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:35:06.664690173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:06.664708916Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:35:06.664718021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:35:07.689128201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:35:07.692735085Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-02-16T14:35:08.053873003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:35:08.056761319Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:35:08.088866183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="490 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:35:08.096364924Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-02-16T14:35:08.098410133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:35:08.100773437Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
ts=2026-02-16T14:35:08.324501719Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:50860"
level=info ts=2026-02-16T14:35:08.775793494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:35:08.778816736Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
ts=2026-02-16T14:35:10.597217985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-77cddbcfd9-8hh5l-04125aa8 10.128.0.73:7946"
level=info ts=2026-02-16T14:35:11.191889611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:35:11.194777961Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=warn ts=2026-02-16T14:35:11.66443474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:12.064614927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:35:12.543429895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:35:12.546623319Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:35:12.998847109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:35:13.00189987Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:35:13.172427584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:35:13.175440635Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-02-16T14:35:14.859220819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:35:14.862259359Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:35:15.935208108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b51c839d6227bed immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"build-e2e-edsn\"}"
level=error ts=2026-02-16T14:35:15.942746081Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b51c839d6227bed/19c66c055c2%3A19c66cd71c4%3A8e0d5b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-edsn\"}"
level=debug ts=2026-02-16T14:35:16.239436476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-02-16T14:35:16.66483569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:17.019961465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:35:17.026039032Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3Aa0602f12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-02-16T14:35:18.44404984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:35:18.447245448Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:35:20.742229281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:35:20.74521127Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-02-16T14:35:21.663994287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:35:26.663998454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:35:31.664443336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:33.317278646Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=info ts=2026-02-16T14:35:33.318320516Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:35:35.755503831Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-02-16T14:35:35.846866141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42448"
level=info ts=2026-02-16T14:35:36.664359971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-02-16T14:35:36.664387852Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:36.664415659Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:35:36.664426508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:35:37.448701481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:35:37.451596001Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-02-16T14:35:38.326313444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.72:40950"
level=info ts=2026-02-16T14:35:39.504952854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:35:39.507942715Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:35:39.552987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:35:39.560161763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:35:39.603154375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:35:39.605975081Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-02-16T14:35:39.961047746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:35:39.963781808Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-02-16T14:35:40.599597753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=info ts=2026-02-16T14:35:40.843286414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:35:40.84645827Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:35:41.047011585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:35:41.050973086Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-02-16T14:35:41.664812095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:35:43.499389389Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-02-16T14:35:45.139103286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T14:35:45.142148002Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66ba48e6%3A19c66ba48e7%3A9da742c5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
ts=2026-02-16T14:35:46.303800949Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.66:48376"
level=warn ts=2026-02-16T14:35:46.664387026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:49.173097875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:35:49.180093857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66ccda21%3Abb35f913?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:35:49.335686635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bc9b90bafea7252c immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="48 kB" avg_uncomp="48 kB" full=1 labels="{stream=\"openshift-service-ca\"}"
level=error ts=2026-02-16T14:35:49.338513678Z caller=flush.go:262 component=ingester loop=12 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bc9b90bafea7252c/19c66ba4569%3A19c66bfa9e3%3A8317114e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca\"}"
level=info ts=2026-02-16T14:35:49.66420172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=72478de770623b49 immediate=false num_chunks=1 total_comp="456 B" avg_comp="456 B" total_uncomp="339 B" avg_uncomp="339 B" full=1 labels="{stream=\"appstudio-probe-monitoring-grafana\"}"
level=error ts=2026-02-16T14:35:49.667160411Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/72478de770623b49/19c66c0b35d%3A19c66c0b35e%3Ada609a61?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"appstudio-probe-monitoring-grafana\"}"
level=info ts=2026-02-16T14:35:51.532027431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=f790566b57097c82 immediate=false num_chunks=1 total_comp="137 kB" avg_comp="137 kB" total_uncomp="668 kB" avg_uncomp="668 kB" full=1 labels="{stream=\"gitlab-rep-ffjb\"}"
level=error ts=2026-02-16T14:35:51.535520407Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/f790566b57097c82/19c66c0559e%3A19c66c954ad%3Ad55fcc2a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-ffjb\"}"
level=info ts=2026-02-16T14:35:51.647735536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:35:51.650640441Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-02-16T14:35:51.664787519Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:52.318564362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-02-16T14:35:52.321574512Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19c66ba48e7%3A19c66ba8159%3Aca76a72f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-02-16T14:35:53.543643527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:35:53.545427496Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
level=info ts=2026-02-16T14:35:55.745423872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=c16f1711c18c7a78 immediate=false num_chunks=1 total_comp="1.7 kB" avg_comp="1.7 kB" total_uncomp="3.5 kB" avg_uncomp="3.5 kB" full=1 labels="{stream=\"konflux-otel\"}"
level=error ts=2026-02-16T14:35:55.74830812Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/c16f1711c18c7a78/19c66ba4589%3A19c66ba458a%3A1def8449?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-otel\"}"
level=info ts=2026-02-16T14:35:56.099317766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="490 MB" avg_uncomp="23 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-02-16T14:35:56.106975963Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19c66b9db81%3A19c66c0181e%3A82c11e48?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-02-16T14:35:56.664725654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:35:58.436431808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="1.8 MB" avg_comp="1.8 MB" total_uncomp="12 MB" avg_uncomp="12 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-02-16T14:35:58.442468266Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19c66b9df34%3A19c66d9a003%3Aa0602f12?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-02-16T14:36:01.664165287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:02.85089127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=9010fa0e5e3d1570 immediate=false num_chunks=1 total_comp="337 B" avg_comp="337 B" total_uncomp="274 B" avg_uncomp="274 B" full=1 labels="{stream=\"openshift-network-operator\"}"
level=error ts=2026-02-16T14:36:02.853858526Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/9010fa0e5e3d1570/19c66ba48e7%3A19c66ba8138%3A7d3e691e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-operator\"}"
level=info ts=2026-02-16T14:36:02.992722215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7bafa8ae3a17fc9a immediate=false num_chunks=1 total_comp="660 B" avg_comp="660 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-ndzq\"}"
level=error ts=2026-02-16T14:36:02.995671903Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7bafa8ae3a17fc9a/19c66bfee6b%3A19c66c01194%3A3877c55b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-ndzq\"}"
level=info ts=2026-02-16T14:36:05.290996024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.2 kB" avg_comp="9.2 kB" total_uncomp="46 kB" avg_uncomp="46 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-02-16T14:36:05.294072871Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19c66b9e16c%3A19c66ba5729%3Aa5d888ef?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-02-16T14:36:06.114782821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=2 total_comp="1.5 kB" avg_comp="736 B" total_uncomp="3.4 kB" avg_uncomp="1.7 kB" full=2 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-02-16T14:36:06.117803941Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19c66ba4589%3A19c66ba8159%3A5b6f11ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-02-16T14:36:06.664351211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:36:06.664402277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:36:06.664410985Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-02-16T14:36:06.664360691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:06.934897516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-02-16T14:36:06.938512108Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19c66b9e19d%3A19c66cedabc%3A5a934c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-02-16T14:36:07.223593579Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-02-16T14:36:07.22526439Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-02-16T14:36:07.403109087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="537 B" avg_comp="537 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-02-16T14:36:07.405867533Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19c66ba48e7%3A19c66ba8159%3A4c36b902?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-02-16T14:36:10.602399162Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a4df028c 10.129.0.66:7946"
level=info ts=2026-02-16T14:36:10.848582117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-02-16T14:36:10.851794807Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19c66ba4589%3A19c66ba857b%3A343887dd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-02-16T14:36:11.558134237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7b51c839d6227bed immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"build-e2e-edsn\"}"
level=error ts=2026-02-16T14:36:11.56591442Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7b51c839d6227bed/19c66c055c2%3A19c66cd71c4%3A8e0d5b7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-edsn\"}"
level=warn ts=2026-02-16T14:36:11.664139009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-02-16T14:36:16.664069829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-02-16T14:36:17.637191439Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-02-16T14:36:21.664060862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:22.648662888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="5.8 kB" avg_comp="5.8 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-02-16T14:36:22.657630994Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19c66ba4589%3A19c66ba458a%3Ab410173c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-02-16T14:36:24.712662368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7f2eee457965d27e immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="2.5 kB" avg_uncomp="2.5 kB" full=1 labels="{stream=\"build-e2e-guaw\"}"
level=error ts=2026-02-16T14:36:24.716422376Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7f2eee457965d27e/19c66c05985%3A19c66c172c7%3Ac8b8e769?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-guaw\"}"
level=info ts=2026-02-16T14:36:25.332161822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-02-16T14:36:25.335538058Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19c66ba4589%3A19c66ba8159%3Aa17be5ee?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-02-16T14:36:26.664262909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:26.868200439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="659 B" avg_comp="659 B" total_uncomp="2.1 kB" avg_uncomp="2.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-02-16T14:36:26.871615293Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19c66ba4910%3A19c66baeb0b%3A2a68c1b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-02-16T14:36:27.505077983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.8 MB" avg_comp="2.4 MB" total_uncomp="42 MB" avg_uncomp="21 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-02-16T14:36:27.512434218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19c66b9dbf5%3A19c66bb0519%3A610d09ca?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-02-16T14:36:28.53765447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="11 kB" avg_comp="11 kB" total_uncomp="35 kB" avg_uncomp="35 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-02-16T14:36:28.540823119Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19c66c030d3%3A19c66c2114c%3A9c31a775?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-02-16T14:36:31.664751636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:31.975321597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-02-16T14:36:31.982607402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19c66b9d8a5%3A19c66ccda21%3Abb35f913?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-02-16T14:36:32.311577115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="63 kB" avg_comp="63 kB" total_uncomp="378 kB" avg_uncomp="378 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-02-16T14:36:32.314900004Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19c66b9df70%3A19c66bb1c0d%3A9a495143?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-02-16T14:36:33.273592358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="639 B" avg_comp="639 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-02-16T14:36:33.277039515Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19c66ba48e6%3A19c66ba48e7%3A9da742c5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-02-16T14:36:33.317719654Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-02-16T14:36:33.317855598Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.128.0.74:11211 count=1
level=warn ts=2026-02-16T14:36:36.664368777Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:36.664366942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-02-16T14:36:36.664431594Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-02-16T14:36:36.664442364Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-02-16T14:36:39.051599957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=feded47fddc003f3 immediate=false num_chunks=1 total_comp="949 B" avg_comp="949 B" total_uncomp="2.4 kB" avg_uncomp="2.4 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator\"}"
level=error ts=2026-02-16T14:36:39.0552093Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/feded47fddc003f3/19c66ba8158%3A19c66ba8159%3Abd595f29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator\"}"
ts=2026-02-16T14:36:40.605199025Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-7b8b49d965-k2gs4-4f1fad35 10.128.0.72:7946"
level=warn ts=2026-02-16T14:36:41.664422241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-02-16T14:36:43.694493936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.6 kB" avg_comp="2.6 kB" total_uncomp="10 kB" avg_uncomp="10 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-02-16T14:36:43.697799815Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19c66b9de43%3A19c66ba5729%3A35149f9b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"