level=info ts=2026-05-11T11:38:11.72151231Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-11T11:38:11.721549436Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-11T11:38:11.722129223Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-11T11:38:11.723439767Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-11T11:38:11.724003824Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:38:11.724094184Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
level=info ts=2026-05-11T11:38:11.724700695Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-28cb5960
ts=2026-05-11T11:38:11.724775315Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-11T11:38:11.724789406Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=debug ts=2026-05-11T11:38:11.72481644Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=info ts=2026-05-11T11:38:11.725024507Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-11T11:38:11.728172771Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T11:38:11.728201578Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-11T11:38:11.72987597Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-11T11:38:11.730144861Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:38:11.730370346Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-11T11:38:11.73038974Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-11T11:38:11.730386954Z
level=debug ts=2026-05-11T11:38:11.730427585Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-11T11:38:11.730476616Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-11T11:38:11.730505658Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-11T11:38:11.730514884Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-11T11:38:11.730513353Z
level=debug ts=2026-05-11T11:38:11.730522654Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-11T11:38:11.731196856Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:38:11.731216142Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:38:11.731227525Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=6.204926ms
level=error ts=2026-05-11T11:38:11.731241504Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-11T11:38:11.731250574Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-11T11:38:11.732882344Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-11T11:38:11.732977928Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
ts=2026-05-11T11:38:11.734616183Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:11.734638154Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:38:11.749753874Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-05-11T11:38:11.749903053Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-11T11:38:11.749931165Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-05-11T11:38:11.7499415Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-11T11:38:11.750250436Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-11T11:38:11.750310996Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-05-11T11:38:11.750348949Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-11T11:38:11.750361371Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-05-11T11:38:11.750367069Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-11T11:38:11.750378387Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=info ts=2026-05-11T11:38:11.750382598Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-11T11:38:11.750390815Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-11T11:38:11.750396493Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-11T11:38:11.750401586Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-11T11:38:11.750406121Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-11T11:38:11.750530391Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-11T11:38:11.750611969Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-11T11:38:11.750630225Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-11T11:38:11.750635951Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-11T11:38:11.750674615Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-11T11:38:11.75069041Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-11T11:38:11.75069622Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-11T11:38:11.750700411Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-05-11T11:38:11.750705503Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-11T11:38:11.750710026Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-11T11:38:11.75072696Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-11T11:38:11.750743328Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T11:38:11.750813252Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=36.437µs success=true
level=info ts=2026-05-11T11:38:11.750834284Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-11T11:38:11.751055233Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=331.481µs errors=false
level=info ts=2026-05-11T11:38:11.751070598Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-11T11:38:11.751208374Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=483.673µs errors=false
level=info ts=2026-05-11T11:38:11.751223876Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-11T11:38:11.7512322Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=508.065µs
level=info ts=2026-05-11T11:38:11.751377905Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-11T11:38:11.751718781Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-11T11:38:11.751808623Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-11T11:38:11.75182999Z caller=loki.go:615 msg="Loki started" startup_time=114.578646ms
level=info ts=2026-05-11T11:38:11.751755996Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-05-11T11:38:11.751996734Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=7.405531518s
level=debug ts=2026-05-11T11:38:11.752054096Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-11T11:38:11.75206256Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-05-11T11:38:11.75213528Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-11T11:38:12.870366026Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:12.870388907Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:38:16.686857101Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:16.686879997Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:38:16.752594905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:38:21.752349748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:38:22.114973006Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:38:22.115001934Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:38:26.039543621Z caller=logging.go:137 msg="GET /ready (503) 124.754µs"
level=warn ts=2026-05-11T11:38:26.752636156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:38:31.75253691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:38:36.037388025Z caller=logging.go:137 msg="GET /ready (503) 96.2µs"
ts=2026-05-11T11:38:36.046957874Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.129.0.79:7946: dial tcp 10.129.0.79:7946: connect: connection refused"
ts=2026-05-11T11:38:36.048163507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.89:7946"
level=info ts=2026-05-11T11:38:36.048754548Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=1 elapsed_time=24.317499327s
level=warn ts=2026-05-11T11:38:36.752725235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:38:41.752118676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-11T11:38:41.752169062Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-11T11:38:41.752176616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:38:41.75218468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:38:42.083966586Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39906"
ts=2026-05-11T11:38:45.192352418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33196"
level=debug ts=2026-05-11T11:38:46.703257006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:38:46.752081246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:38:51.752752791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:38:56.752368822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:38:59.007495362Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:39:01.751951616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:39:06.752156577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:09.163189348Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:39:11.729860042Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:39:11.730926305Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:39:11.752389933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:39:11.752396387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:11.75241758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:39:11.752425144Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:39:16.752400639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:19.596047991Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:39:19.596111821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:39:19.606911264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:39:21.752762905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:39:26.752581837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:26.905723887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:39:29.009127932Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T11:39:31.751880032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:39:36.751896244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:39:37.743426214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:39:37.750731121Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T11:39:41.752346947Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:39:41.752380526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:39:41.752389324Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:39:41.752389963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:42.37991741Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-11T11:39:45.195293568Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48366"
level=warn ts=2026-05-11T11:39:46.75238498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:39:46.945430332Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:39:51.752293475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:39:56.752338631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:39:59.010163435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:40:01.752203286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:40:06.752361684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:40:08.626725873Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-11T11:40:11.730189631Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:40:11.730224876Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:40:11.752780124Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:40:11.752776304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:40:11.752807002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:40:11.752815465Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T11:40:12.088954654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:50546"
ts=2026-05-11T11:40:15.082855475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43922"
ts=2026-05-11T11:40:15.196422044Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48768"
level=debug ts=2026-05-11T11:40:15.593282733Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:40:16.557171392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:40:16.567560738Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:40:16.752840526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:40:21.752035041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:40:26.752956814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:40:29.012177527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:40:31.752415433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:40:34.900325254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:40:34.902133946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:40:36.752792683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:40:41.752092376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:40:41.752095791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:40:41.752122768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:40:41.752130481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T11:40:42.090660502Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39852"
ts=2026-05-11T11:40:45.198180559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36930"
level=warn ts=2026-05-11T11:40:46.752900973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:40:48.177670524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:40:51.752684037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:40:56.752192969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:40:59.013120194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:41:01.752456041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:41:01.988317868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:41:01.99746682Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:41:06.752625733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:41:11.729713841Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:41:11.730780622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:41:11.752826846Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:41:11.752865065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:41:11.752876014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:41:11.752874739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:41:12.092276131Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43430"
level=debug ts=2026-05-11T11:41:13.265617674Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T11:41:15.086732768Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54416"
ts=2026-05-11T11:41:15.199700791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52168"
level=debug ts=2026-05-11T11:41:15.532934139Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:41:16.75191552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:41:21.751916974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:41:26.752275519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:41:29.01452471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T11:41:31.752779621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:41:36.752730203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:41:41.752589299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:41:41.752618513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:41:41.752626663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:41:41.752635988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:41:42.041531692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:41:46.752391189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:41:51.751918809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:41:54.884423296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:41:54.891617562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:41:56.752615629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:41:59.015931182Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:42:01.752705629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:42:03.091526376Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:42:06.752160274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:42:07.056524164Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:42:11.729855573Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:42:11.730903524Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:42:11.752780946Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:42:11.752811126Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:42:11.752820822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:42:11.752839751Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:42:16.752227647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:42:21.752022495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:42:26.752292921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:42:29.017514369Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T11:42:31.752233017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:42:33.194359042Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:42:36.752657277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:42:41.751931899Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:42:41.751962033Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:42:41.751970736Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:42:41.751967829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:42:42.097777222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45460"
ts=2026-05-11T11:42:45.092102724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49964"
ts=2026-05-11T11:42:45.203893126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49604"
level=warn ts=2026-05-11T11:42:46.752585633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:42:48.053506725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:42:48.061006639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:42:51.751867531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:42:53.970412366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:42:56.752126585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:42:59.019125313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:43:01.721148953Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:43:01.75230686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:43:06.751937216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:43:11.730701605Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:43:11.731530552Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:43:11.752254988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:43:11.752284717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:43:11.752292507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T11:43:11.752285671Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-11T11:43:11.752312863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:43:11.752444141Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-11T11:43:11.752452846Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T11:43:11.75246646Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.274µs success=true
level=debug ts=2026-05-11T11:43:11.752682537Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=debug ts=2026-05-11T11:43:16.10773183Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-11T11:43:16.752608792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:20.462845816Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-11T11:43:21.752892512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:24.818284595Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-05-11T11:43:26.752663122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:43:29.021085607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:43:29.172074429Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-11T11:43:31.752731806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:33.527463729Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-11T11:43:35.197453517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:43:35.206034429Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:43:36.752174863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:37.580479313Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:43:37.882626373Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-11T11:43:41.752832573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:43:41.752845978Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:43:41.752868599Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:43:41.752876182Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T11:43:42.101194582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56250"
level=debug ts=2026-05-11T11:43:42.237602157Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=debug ts=2026-05-11T11:43:42.237626072Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=8
ts=2026-05-11T11:43:45.094517353Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33870"
ts=2026-05-11T11:43:45.206654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37886"
level=debug ts=2026-05-11T11:43:46.591456306Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-11T11:43:46.75276741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:50.946322331Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=warn ts=2026-05-11T11:43:51.752347521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:43:55.301330852Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=debug ts=2026-05-11T11:43:55.449269421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-11T11:43:56.752761575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:43:59.022182559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:43:59.656625612Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-11T11:44:01.752631187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:04.011079638Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T11:44:06.752731475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:08.365829465Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-11T11:44:09.820603834Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:44:11.73055737Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:44:11.731670282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:44:11.752371528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:44:11.752398233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:44:11.752406222Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:44:11.752440859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:44:12.102381412Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57406"
level=debug ts=2026-05-11T11:44:12.721105794Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-11T11:44:16.752433356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:17.07574551Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-11T11:44:21.430841296Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-11T11:44:21.751966304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:25.785839574Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T11:44:26.752056475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:44:29.023331567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:44:30.140479903Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-11T11:44:31.752519408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:44:31.815633599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:44:31.825900944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:44:34.495105953Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-11T11:44:36.752020118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:38.849523158Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-11T11:44:41.751992126Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:44:41.751992369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:41.752019259Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:44:41.752027204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:44:43.204666749Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
ts=2026-05-11T11:44:45.097394169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40486"
ts=2026-05-11T11:44:45.209476079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:58980"
level=debug ts=2026-05-11T11:44:45.749984026Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:44:46.752211616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:47.560171778Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=debug ts=2026-05-11T11:44:47.560194548Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-05-11T11:44:51.752847424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:44:51.914151675Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=debug ts=2026-05-11T11:44:56.269825588Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-11T11:44:56.752819227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:44:59.024806783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:45:00.623907139Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T11:45:01.751904637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:03.042124485Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-11T11:45:03.044035154Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T11:45:04.978975806Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T11:45:06.751916337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:09.334298344Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-11T11:45:11.729479643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:45:11.730527992Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:45:11.731821537Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T11:45:11.731865028Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976110 n=1
level=debug ts=2026-05-11T11:45:11.73190778Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976110 n=1
level=info ts=2026-05-11T11:45:11.752487476Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:45:11.752514921Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:45:11.752523356Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:45:11.752558818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:13.689162353Z caller=checkpoint.go:372 msg="writing series" size="298 kB"
level=warn ts=2026-05-11T11:45:16.752063676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:17.965738123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:45:18.044053902Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T11:45:21.752108448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:22.398622652Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T11:45:22.448858153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:45:22.459001841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:45:26.751990978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:26.753084865Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-11T11:45:29.025982224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:45:31.107655755Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-11T11:45:31.752291481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:35.462952939Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T11:45:36.752174168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:39.817442763Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=info ts=2026-05-11T11:45:41.752338618Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:45:41.752373001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:45:41.752380743Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:45:41.752377968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:44.172635596Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-11T11:45:46.752261213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:47.003592885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:45:48.528905014Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=debug ts=2026-05-11T11:45:48.528932452Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-11T11:45:51.75288571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:52.882519362Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T11:45:56.229161794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T11:45:56.75192927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:45:57.237010321Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
ts=2026-05-11T11:45:59.027532444Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:46:01.592371364Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-11T11:46:01.752048997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:05.947156081Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-11T11:46:06.752480454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:10.301468832Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-11T11:46:11.730372148Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:46:11.730380122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:46:11.752208455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:46:11.752238022Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:46:11.752249271Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:46:11.752270834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:13.158464504Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:46:14.656249287Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-11T11:46:16.751993585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:46:17.479897452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:46:17.486870346Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:46:17.486900937Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T11:46:17.486926685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:46:17.493513134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:46:19.011654846Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-05-11T11:46:21.752390312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:23.366172411Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T11:46:26.7526889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:27.72088901Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T11:46:29.028944148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T11:46:30.383629899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:46:30.394632709Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:46:31.752847447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:32.075598643Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-11T11:46:36.430890626Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-11T11:46:36.752324778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:39.842502609Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:46:40.785817539Z caller=checkpoint.go:372 msg="writing series" size="389 kB"
level=info ts=2026-05-11T11:46:41.751997533Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:46:41.75203084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:46:41.752039554Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:46:41.752034223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:46:42.111946327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40496"
level=debug ts=2026-05-11T11:46:45.140589936Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
ts=2026-05-11T11:46:45.215545419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35432"
level=debug ts=2026-05-11T11:46:46.299883198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:46:46.752648585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:49.494826392Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-11T11:46:49.49485509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=14
level=warn ts=2026-05-11T11:46:51.752504687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:53.849890731Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T11:46:56.752871697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:46:58.204648819Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
ts=2026-05-11T11:46:59.030839079Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T11:47:01.752546653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:47:02.089548145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:47:02.099538991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:47:02.559178273Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=warn ts=2026-05-11T11:47:06.752090864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:06.914537782Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-11T11:47:11.269949841Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T11:47:11.269978773Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=info ts=2026-05-11T11:47:11.730329351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:47:11.730327412Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:47:11.752069573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:47:11.752099479Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:47:11.752109431Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:47:11.752129992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:11.892554776Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:47:15.624497716Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-11T11:47:16.751932788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:19.978714653Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T11:47:21.752025359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:24.334285066Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-11T11:47:26.752390939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:28.688265467Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
ts=2026-05-11T11:47:29.032112313Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:47:30.459357625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T11:47:31.7525844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:33.043683772Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-05-11T11:47:36.752201744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:47:37.403483134Z caller=checkpoint.go:372 msg="writing series" size="4.9 MB"
level=debug ts=2026-05-11T11:47:37.403508623Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.5 MB" series=6
level=debug ts=2026-05-11T11:47:39.216476734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:47:41.75230378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:47:41.752334644Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:47:41.752342352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:47:41.75233742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:47:41.76210339Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-11T11:47:41.765984767Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.013369326s
ts=2026-05-11T11:47:45.105503912Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48776"
ts=2026-05-11T11:47:45.218213333Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33780"
level=warn ts=2026-05-11T11:47:46.752478594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:47:51.752154834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:47:56.752676047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:47:58.3726223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:47:58.383626208Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T11:47:59.033351559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:48:01.752042266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:48:06.752658462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:11.73017835Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:48:11.730218943Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:48:11.751609212Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T11:48:11.751794261Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-11T11:48:11.751807226Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T11:48:11.751805974Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:48:11.751828415Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:48:11.751836583Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T11:48:11.751834637Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.532µs success=true
level=warn ts=2026-05-11T11:48:11.751865677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:11.752064088Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=debug ts=2026-05-11T11:48:13.115654132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:48:15.971069232Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-11T11:48:16.752653549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:20.189987259Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-11T11:48:21.752264806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:24.408558022Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-05-11T11:48:26.75209432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:27.063611822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:48:28.628065505Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
ts=2026-05-11T11:48:29.034658187Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:48:31.752953704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:32.846354451Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-11T11:48:36.752841183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:37.064657111Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=debug ts=2026-05-11T11:48:40.19912413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:48:40.199195973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:48:40.21192931Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:48:41.286728871Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-11T11:48:41.286752657Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=8
level=info ts=2026-05-11T11:48:41.752397685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:48:41.752435087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:48:41.752446748Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:48:41.752509691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:45.502555736Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-11T11:48:46.752773859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:48:47.957932482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:48:47.966519284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:48:49.721258764Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=warn ts=2026-05-11T11:48:51.752775687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:48:52.104478422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:48:52.112906854Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:48:53.939693809Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=warn ts=2026-05-11T11:48:56.752392035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:48:58.159232927Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-11T11:48:59.036024446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T11:49:01.752641588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:02.377131092Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-11T11:49:06.596322275Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-11T11:49:06.752923578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:10.814718694Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=debug ts=2026-05-11T11:49:11.729712794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:49:11.730780899Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:49:11.751986011Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:49:11.752015566Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:49:11.75202413Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:49:11.752040311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:12.818174097Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:49:12.819979591Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T11:49:14.58819949Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:49:15.033666692Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T11:49:16.752713621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:49:18.431296381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:49:18.443009661Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:49:19.252380875Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-05-11T11:49:21.752163532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:23.471795362Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=debug ts=2026-05-11T11:49:23.471825035Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-05-11T11:49:26.752621068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:27.690586058Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-11T11:49:28.618875761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:49:28.628697262Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T11:49:29.037378458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T11:49:31.75193153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:31.909367877Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T11:49:36.127373323Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T11:49:36.752663954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:40.346519943Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=info ts=2026-05-11T11:49:41.751944048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:49:41.751987663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:49:41.751998853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:49:41.751988642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:44.56502867Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
ts=2026-05-11T11:49:45.224366927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48220"
level=warn ts=2026-05-11T11:49:46.752693584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:47.43958997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:49:48.783985958Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=warn ts=2026-05-11T11:49:51.751952543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:53.00240488Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-11T11:49:53.002428694Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T11:49:56.752074184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:49:57.221500865Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-11T11:49:59.039104679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:50:00.14525305Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:50:01.440015943Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T11:50:01.752267406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:05.658505712Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-11T11:50:06.752920654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:50:09.019731181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="71 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:50:09.02891161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:50:09.877731905Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=info ts=2026-05-11T11:50:11.730381657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:50:11.730631834Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-11T11:50:11.751943981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:50:11.752011201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:50:11.752025503Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:50:11.752035656Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:50:14.09666671Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-05-11T11:50:14.919252884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:50:16.752282088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:50:16.868418423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:50:16.880159795Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:50:18.314908105Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T11:50:21.752179168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:22.533402615Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=warn ts=2026-05-11T11:50:26.75237298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:26.752404692Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-05-11T11:50:29.040708212Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:50:30.970941783Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T11:50:31.752311548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:35.189622037Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=warn ts=2026-05-11T11:50:36.752185395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:39.408657589Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=warn ts=2026-05-11T11:50:41.752018592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:50:41.752012868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:50:41.752054694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:50:41.752063154Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:50:42.402936241Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:50:43.497515989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:50:43.628241742Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
ts=2026-05-11T11:50:45.115711008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40708"
level=warn ts=2026-05-11T11:50:46.752856351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:47.84667283Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-11T11:50:51.752695486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:50:52.064855661Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=debug ts=2026-05-11T11:50:52.064890154Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-05-11T11:50:56.284180244Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-11T11:50:56.752702245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:50:58.436057842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:50:58.447472553Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T11:50:59.042729822Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:51:00.50281362Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=warn ts=2026-05-11T11:51:01.75216226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:51:04.716139028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.7 MB" avg_comp="2.7 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:51:04.721795322Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=error ts=2026-05-11T11:51:04.725953625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:51:06.752030897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:07.776944189Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:51:08.940237418Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-11T11:51:11.730289925Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:51:11.730978488Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:51:11.752440818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:51:11.752478478Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:51:11.752490989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:51:11.752530821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:13.158756328Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-11T11:51:15.231461004Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:46706"
level=warn ts=2026-05-11T11:51:16.752273112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:17.377348427Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-11T11:51:20.069450191Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:51:21.597070006Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T11:51:21.752778108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:25.815678691Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=warn ts=2026-05-11T11:51:26.752443852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:51:29.044279321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:51:30.034309759Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-11T11:51:31.751990457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:34.253084903Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-05-11T11:51:36.752106187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:38.471914187Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-05-11T11:51:38.471937977Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-05-11T11:51:41.751898815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:51:41.751891643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:51:41.751930432Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:51:41.751942372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:51:42.612989219Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:51:42.690434302Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
ts=2026-05-11T11:51:45.119071754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40236"
ts=2026-05-11T11:51:45.233041371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33296"
level=warn ts=2026-05-11T11:51:46.751934788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:46.908687521Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
level=debug ts=2026-05-11T11:51:51.127213339Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-11T11:51:51.382993142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:51:51.392149841Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T11:51:51.751864779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:51:55.346813924Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=debug ts=2026-05-11T11:51:55.3468407Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=warn ts=2026-05-11T11:51:56.752628082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:51:59.04611284Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:51:59.565624598Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-05-11T11:52:01.710720604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:52:01.722293773Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:52:01.752368193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:03.758169753Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:52:03.784266565Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T11:52:06.752826579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:08.004319011Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-11T11:52:08.00434558Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
level=debug ts=2026-05-11T11:52:08.032157356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:52:11.730176762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:52:11.730209354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:52:11.752677013Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:52:11.7526874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:11.75270582Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:52:11.752714071Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:52:12.221815869Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-11T11:52:16.440440757Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T11:52:16.752463114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:20.658496049Z caller=checkpoint.go:372 msg="writing series" size="31 kB"
level=warn ts=2026-05-11T11:52:21.75272401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:24.877985579Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-11T11:52:26.752701653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:52:29.047839982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:52:29.096692587Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=warn ts=2026-05-11T11:52:31.752671208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:33.314744167Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-11T11:52:36.751878936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:37.537482186Z caller=checkpoint.go:372 msg="writing series" size="5.5 MB"
level=debug ts=2026-05-11T11:52:37.537504428Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.0 MB" series=7
level=debug ts=2026-05-11T11:52:40.924704143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:52:41.207626882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:52:41.209627553Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T11:52:41.752401539Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:52:41.752421842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:52:41.752440121Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:52:41.752451021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T11:52:41.865784921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="118 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:52:41.87311394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T11:52:41.930949911Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-11T11:52:41.933494389Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.181489802s
level=warn ts=2026-05-11T11:52:46.752467492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:52:51.751876149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:52:54.255275296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:52:54.267916893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:52:54.267954211Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T11:52:54.267988661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:52:54.277829953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:52:56.752075295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:52:59.049432721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T11:53:01.752110175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:53:06.752607217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:08.81819816Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:53:11.730591492Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:53:11.731476403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:53:11.752260103Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T11:53:11.752320338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:53:11.752344852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:53:11.752353632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:53:11.752353271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:53:11.752588534Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-11T11:53:11.752599041Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T11:53:11.752619819Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.048µs success=true
level=debug ts=2026-05-11T11:53:11.752820167Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-11T11:53:12.148774143Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51538"
level=info ts=2026-05-11T11:53:14.16977467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:53:14.183586501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T11:53:15.125659751Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35850"
level=debug ts=2026-05-11T11:53:15.214499977Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-11T11:53:16.752850641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:18.676737Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-11T11:53:21.752874313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:22.138444323Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T11:53:25.599397705Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-11T11:53:26.752755957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:53:29.050776865Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:53:29.06155044Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-11T11:53:31.752686963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:32.523032761Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-11T11:53:35.614198837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:53:35.623136119Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:53:35.984310474Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-11T11:53:36.752349472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:39.445613981Z caller=checkpoint.go:372 msg="writing series" size="9.1 kB"
level=info ts=2026-05-11T11:53:40.543018111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:53:40.550127086Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:53:40.793673746Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T11:53:41.752074739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:53:41.752107089Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:53:41.752115111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:53:41.752125671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:42.030558299Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:53:42.919704406Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T11:53:42.919727637Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=10
ts=2026-05-11T11:53:45.239153006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35628"
level=debug ts=2026-05-11T11:53:46.369010484Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-11T11:53:46.752397919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:49.830701035Z caller=checkpoint.go:372 msg="writing series" size="447 kB"
level=warn ts=2026-05-11T11:53:51.751870593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:53.291716266Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
level=warn ts=2026-05-11T11:53:56.752559929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:53:56.753672267Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=debug ts=2026-05-11T11:53:56.753694623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-11T11:53:59.051764733Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:54:00.21516073Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T11:54:01.752776968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:03.676933666Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=warn ts=2026-05-11T11:54:06.752404379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:07.137927551Z caller=checkpoint.go:372 msg="writing series" size="442 kB"
level=debug ts=2026-05-11T11:54:07.447973309Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:54:10.599939782Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-11T11:54:11.730217381Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:54:11.730687083Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:54:11.752398934Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:54:11.752429316Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T11:54:11.752417547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:54:11.752437213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T11:54:12.15197701Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:40806"
level=debug ts=2026-05-11T11:54:14.06074119Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T11:54:16.752835385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:17.522941349Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-11T11:54:20.984329601Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T11:54:21.752444452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:23.403274736Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:54:24.445844813Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-11T11:54:26.752495809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:27.907293081Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-11T11:54:27.907325874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-05-11T11:54:29.052953101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T11:54:29.977450909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:54:29.985467379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:54:31.368457997Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-11T11:54:31.752103632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:34.830046395Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=warn ts=2026-05-11T11:54:36.752907382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:37.309628043Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:54:37.952282161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:54:37.961625692Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:54:38.291488868Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-11T11:54:41.752089684Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:54:41.75209773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:41.752123523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:54:41.752143048Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:54:41.753349853Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-11T11:54:45.215015183Z caller=checkpoint.go:372 msg="writing series" size="225 kB"
ts=2026-05-11T11:54:45.242250794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:45822"
level=warn ts=2026-05-11T11:54:46.752480319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:48.676345044Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T11:54:51.752040474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:54:52.138347665Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-11T11:54:55.599516652Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T11:54:56.751995656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:54:59.054635007Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:54:59.061462377Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=debug ts=2026-05-11T11:54:59.061484691Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-11T11:55:01.752144793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:02.522471403Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-11T11:55:04.041529275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:55:05.983927906Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-11T11:55:06.751884353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:09.44553214Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=info ts=2026-05-11T11:55:11.263591658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:55:11.270719891Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:55:11.730129982Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:55:11.73118836Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:55:11.752077754Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:55:11.752112171Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:55:11.752120618Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:55:11.752112706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:12.907025766Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-11T11:55:13.562332499Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:55:15.131704214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45594"
level=debug ts=2026-05-11T11:55:16.369056111Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T11:55:16.752607182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:55:19.321200642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:55:19.331093107Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:55:19.331119583Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T11:55:19.33114858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:55:19.339715679Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:55:19.830232782Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T11:55:21.752488079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:23.291762801Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-11T11:55:26.752274628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:26.753388861Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-05-11T11:55:29.055721644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:55:30.214784915Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=warn ts=2026-05-11T11:55:31.752534664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:32.603402175Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:55:33.676681366Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-05-11T11:55:35.729436225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:55:35.740346971Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T11:55:36.752859736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:37.138392381Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-11T11:55:40.599665968Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-11T11:55:41.752174594Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:55:41.752214105Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:55:41.752224138Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:55:41.75223329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:44.061505595Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
ts=2026-05-11T11:55:45.246009715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55058"
level=warn ts=2026-05-11T11:55:46.752783287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:47.522522261Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T11:55:50.984469585Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-11T11:55:51.752099989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:54.145365605Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:55:54.446096418Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-11T11:55:54.44611657Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-05-11T11:55:56.752333616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:55:57.907173274Z caller=checkpoint.go:372 msg="writing series" size="514 B"
ts=2026-05-11T11:55:59.056836153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T11:55:59.670079583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="126 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:55:59.677344763Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:56:01.278024824Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:56:01.368415437Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=warn ts=2026-05-11T11:56:01.751960883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:04.830417462Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=warn ts=2026-05-11T11:56:06.752187926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:08.291500928Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-11T11:56:09.528270217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:56:09.537941836Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:56:11.730227395Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:56:11.730233731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:56:11.752729345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:56:11.752760637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:56:11.752768429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:56:11.752795577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:11.752834135Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-11T11:56:15.215111176Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-11T11:56:16.752752154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:18.67639365Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T11:56:21.752729148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:22.138020778Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=debug ts=2026-05-11T11:56:25.599770364Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T11:56:26.752225116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:56:29.058406518Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:56:29.061740641Z caller=checkpoint.go:372 msg="writing series" size="415 kB"
level=debug ts=2026-05-11T11:56:29.489174307Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:56:31.752174478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:32.522500387Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-11T11:56:35.984482325Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-11T11:56:36.752316185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:39.445925803Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=debug ts=2026-05-11T11:56:39.44595036Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=info ts=2026-05-11T11:56:41.752777904Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:56:41.752808472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:56:41.752816971Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:56:41.752833973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:42.907301769Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
ts=2026-05-11T11:56:45.136724369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56354"
level=info ts=2026-05-11T11:56:46.154516525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="149 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:56:46.166286148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:56:46.368754957Z caller=checkpoint.go:372 msg="writing series" size="753 kB"
level=debug ts=2026-05-11T11:56:46.368779279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-11T11:56:46.752684538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:49.830507549Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-11T11:56:50.545949764Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T11:56:50.547954966Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T11:56:51.752707159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:53.291469389Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T11:56:56.752211816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:56:56.753501934Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
ts=2026-05-11T11:56:59.059713043Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:57:00.215024021Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T11:57:01.037004875Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:57:01.752535019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:03.676462591Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T11:57:06.752200015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:07.139625469Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-11T11:57:07.139653319Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
level=info ts=2026-05-11T11:57:09.109273367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:57:09.11854758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:57:10.599708159Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=debug ts=2026-05-11T11:57:11.73001509Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:57:11.731069281Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T11:57:11.75282537Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:57:11.752862604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:57:11.752870936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T11:57:11.752938942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:14.061060888Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T11:57:15.13811681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50750"
ts=2026-05-11T11:57:15.250639705Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41526"
level=warn ts=2026-05-11T11:57:16.752528763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:17.522676622Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-05-11T11:57:20.984478437Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-11T11:57:21.752840334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:24.446173653Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=debug ts=2026-05-11T11:57:25.210340234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:57:26.752405548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:27.907149335Z caller=checkpoint.go:372 msg="writing series" size="387 kB"
ts=2026-05-11T11:57:29.061190515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T11:57:31.368804362Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-05-11T11:57:31.752184939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:34.836453772Z caller=checkpoint.go:372 msg="writing series" size="6.3 MB"
level=debug ts=2026-05-11T11:57:34.836482587Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.1 MB" series=8
level=warn ts=2026-05-11T11:57:36.752518196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:38.292140919Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=debug ts=2026-05-11T11:57:41.752849499Z caller=checkpoint.go:384 msg="flushing series" totalSize="257 kB" series=1
level=info ts=2026-05-11T11:57:41.752875656Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:57:41.752894215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:41.75291011Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:57:41.752919008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T11:57:41.792170411Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-11T11:57:41.795994384Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.043219035s
level=info ts=2026-05-11T11:57:41.874241312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="172 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:57:41.883139876Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T11:57:45.139288553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60326"
level=warn ts=2026-05-11T11:57:46.752854252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:57:49.753844189Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:57:50.369035878Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T11:57:51.193523482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:57:51.201981852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:57:51.752508896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:57:56.752519452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:57:59.062348166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T11:58:01.752755066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T11:58:06.752689875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:58:11.730759295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:58:11.7313297Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:58:11.751707221Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T11:58:11.751909324Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-11T11:58:11.75192065Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T11:58:11.751945283Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.745µs success=true
level=warn ts=2026-05-11T11:58:11.751953749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:58:11.751954679Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:58:11.75200041Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:58:11.752009924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:58:11.760857971Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-11T11:58:12.163452861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57994"
level=debug ts=2026-05-11T11:58:15.223113292Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=warn ts=2026-05-11T11:58:16.752827394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:17.918188382Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:58:18.684698617Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-11T11:58:21.752180235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:22.14661414Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T11:58:25.607928895Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=warn ts=2026-05-11T11:58:26.752313871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:29.05585045Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T11:58:29.064172382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:58:29.069006281Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-11T11:58:30.765539358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="172 MB" avg_uncomp="22 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:58:30.773074141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T11:58:31.752259476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:32.530666258Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-11T11:58:35.992062998Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-11T11:58:36.752472527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:39.453860886Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-11T11:58:39.453890768Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-11T11:58:41.752445596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:58:41.752463303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T11:58:41.752493454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:58:41.752503597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:58:42.939403686Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-11T11:58:42.939441973Z caller=checkpoint.go:384 msg="flushing series" totalSize="24 MB" series=1
level=debug ts=2026-05-11T11:58:43.673410629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T11:58:45.143046602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34186"
level=debug ts=2026-05-11T11:58:46.377137887Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=warn ts=2026-05-11T11:58:46.752068471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:58:47.628347889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:58:47.636328127Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:58:49.83850025Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=warn ts=2026-05-11T11:58:51.75246928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:53.302622248Z caller=checkpoint.go:372 msg="writing series" size="934 kB"
level=debug ts=2026-05-11T11:58:53.302651847Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-05-11T11:58:56.752344224Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:58:56.761474317Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-05-11T11:58:59.06613895Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T11:59:00.223426171Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T11:59:01.751886987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:03.684958828Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=warn ts=2026-05-11T11:59:06.752697288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:07.147079108Z caller=checkpoint.go:372 msg="writing series" size="942 kB"
level=debug ts=2026-05-11T11:59:07.147112264Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-11T11:59:09.98057828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T11:59:10.607834113Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T11:59:11.730355432Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T11:59:11.730867086Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T11:59:11.752231802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:59:11.752241097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:11.752277506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:59:11.752289909Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:59:14.068764026Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T11:59:16.752636403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:17.530874839Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-11T11:59:20.991921917Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=warn ts=2026-05-11T11:59:21.752845673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:24.453976151Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-11T11:59:26.752772415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:27.91510577Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
ts=2026-05-11T11:59:29.068142619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T11:59:29.462708927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="196 MB" avg_uncomp="22 MB" full=8 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T11:59:29.474101781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T11:59:29.81122175Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T11:59:31.376652385Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-11T11:59:31.751941206Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:34.838310503Z caller=checkpoint.go:372 msg="writing series" size="6.4 kB"
level=debug ts=2026-05-11T11:59:35.505635766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T11:59:36.752030697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T11:59:36.775235332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:59:36.783145679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:59:36.783176519Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T11:59:36.783210581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:59:36.790931267Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T11:59:38.300156442Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-11T11:59:41.75260959Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T11:59:41.752635215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:41.75264684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T11:59:41.752656267Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T11:59:41.761955005Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=debug ts=2026-05-11T11:59:41.761985218Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
ts=2026-05-11T11:59:45.149193973Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55176"
level=debug ts=2026-05-11T11:59:45.223068696Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=warn ts=2026-05-11T11:59:46.752554636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:48.685024708Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-05-11T11:59:50.060625084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T11:59:50.070342702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T11:59:51.752542095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T11:59:52.146107274Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-11T11:59:55.607754807Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T11:59:56.752168073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T11:59:59.069693343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T11:59:59.069764134Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=warn ts=2026-05-11T12:00:01.752821151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:02.530365539Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=debug ts=2026-05-11T12:00:02.53039116Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-11T12:00:05.992026367Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:00:06.752616981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:09.454182974Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-05-11T12:00:10.595740442Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:00:11.730809181Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:00:11.731963253Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T12:00:11.732025429Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976111 n=1
level=debug ts=2026-05-11T12:00:11.732095735Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976111 n=1
level=debug ts=2026-05-11T12:00:11.732250409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:00:11.752618648Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:00:11.75265277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:00:11.752662639Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:00:11.752683362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:12.915197876Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-11T12:00:15.151057398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43772"
level=info ts=2026-05-11T12:00:16.073056702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="27 MB" avg_comp="2.7 MB" total_uncomp="219 MB" avg_uncomp="22 MB" full=9 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:00:16.084847464Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:00:16.376680091Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T12:00:16.752905123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:19.83945046Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-11T12:00:20.479639373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:00:21.751982682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:23.301283433Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-11T12:00:23.301314204Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=6
level=info ts=2026-05-11T12:00:25.769427441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:00:25.778224451Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:00:26.752479954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:26.761625532Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:00:29.071876214Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:00:30.223879328Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-05-11T12:00:31.752448473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:33.68462292Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T12:00:36.75230932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:37.146279683Z caller=checkpoint.go:372 msg="writing series" size="7.2 kB"
level=debug ts=2026-05-11T12:00:40.607212918Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=warn ts=2026-05-11T12:00:41.752310193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:00:41.752355341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:00:41.752369283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:00:41.75237942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:00:42.408735613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:00:44.070363156Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=warn ts=2026-05-11T12:00:46.752366061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:47.530630397Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:00:50.992341473Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=debug ts=2026-05-11T12:00:50.992374156Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-11T12:00:51.752160903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:54.45413707Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-05-11T12:00:56.7519008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:00:57.915841314Z caller=checkpoint.go:372 msg="writing series" size="514 B"
ts=2026-05-11T12:00:59.073653855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:01:01.377285157Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-11T12:01:01.753184976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:03.748343901Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:01:03.759035128Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:01:04.837879215Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-11T12:01:05.092869713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="30 MB" avg_comp="2.7 MB" total_uncomp="243 MB" avg_uncomp="22 MB" full=10 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:01:05.127442718Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:01:06.753004327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:08.300047673Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-11T12:01:11.730394445Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:01:11.737038183Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-11T12:01:11.75241376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:01:11.752447159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:01:11.752462414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:01:11.752474753Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:01:11.761919055Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-05-11T12:01:13.968238741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:01:13.969898571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-11T12:01:13.981562896Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:01:15.222746042Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-11T12:01:16.752677527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:18.684521337Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:01:21.752382597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:22.146068976Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-05-11T12:01:25.607701044Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T12:01:26.753043539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:29.06909262Z caller=checkpoint.go:372 msg="writing series" size="654 kB"
level=debug ts=2026-05-11T12:01:29.069121082Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-11T12:01:29.075200815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:01:31.752706562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:32.530688856Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-11T12:01:35.9923042Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=warn ts=2026-05-11T12:01:36.75273486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:39.454248715Z caller=checkpoint.go:372 msg="writing series" size="675 kB"
level=info ts=2026-05-11T12:01:41.752361513Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:01:41.752396185Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:01:41.752405651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:01:41.752411976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:42.916026505Z caller=checkpoint.go:372 msg="writing series" size="600 kB"
level=debug ts=2026-05-11T12:01:42.916050245Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-11T12:01:45.156351823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33574"
ts=2026-05-11T12:01:45.267128294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:60504"
level=debug ts=2026-05-11T12:01:46.377322507Z caller=checkpoint.go:372 msg="writing series" size="880 kB"
level=warn ts=2026-05-11T12:01:46.752755649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:49.060222564Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:01:49.838646868Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:01:51.752172084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:52.152152766Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:01:53.299921067Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T12:01:56.752758196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:01:56.762068756Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=debug ts=2026-05-11T12:01:56.762095065Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-11T12:01:59.077303902Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:02:00.223525161Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-11T12:02:01.752530994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:02:02.99100293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="33 MB" avg_comp="2.7 MB" total_uncomp="266 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:02:03.005451179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:02:03.005479548Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:02:03.005515315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="33 MB" avg_comp="2.7 MB" total_uncomp="266 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:02:03.01308573Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:02:03.684102202Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T12:02:06.75219147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:07.15055556Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-11T12:02:07.150597261Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=3
level=debug ts=2026-05-11T12:02:10.608001433Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=debug ts=2026-05-11T12:02:11.73062533Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:02:11.730631398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:02:11.752278936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:02:11.752314875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:02:11.75232447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:02:11.752324397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:02:13.626586986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:02:13.638998843Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:02:14.069363618Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-11T12:02:14.653163496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="33 MB" avg_comp="2.7 MB" total_uncomp="266 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:02:14.661819441Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:02:16.752179505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:17.414330745Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:02:17.530895124Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-11T12:02:20.992267384Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-11T12:02:21.752431184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:24.453962372Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:02:26.752313099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:27.915047252Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
ts=2026-05-11T12:02:29.079205743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:02:31.377024125Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-11T12:02:31.751874685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:34.845895127Z caller=checkpoint.go:372 msg="writing series" size="6.8 MB"
level=debug ts=2026-05-11T12:02:34.845920775Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=8
level=warn ts=2026-05-11T12:02:36.752626489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:38.300005024Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=info ts=2026-05-11T12:02:39.805650274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="33 MB" avg_comp="2.7 MB" total_uncomp="266 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:02:39.813053967Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:02:41.751818591Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:02:41.751851071Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:02:41.751859216Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:02:41.75187282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:02:41.761058092Z caller=checkpoint.go:384 msg="flushing series" totalSize="530 kB" series=1
level=info ts=2026-05-11T12:02:41.78371739Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-11T12:02:41.785104246Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.024296391s
ts=2026-05-11T12:02:42.176799821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48104"
level=debug ts=2026-05-11T12:02:42.221889914Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:02:46.287453277Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:02:46.287541553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:02:46.293604688Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:02:46.752270725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:02:51.752799382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:02:56.752092422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:02:58.174254506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:02:58.182433588Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-11T12:02:59.080464578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:03:01.75201643Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:03:02.653624381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:03:02.662025499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:03:06.752388957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:03:11.73031915Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:03:11.730379605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:03:11.751602754Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:03:11.751762636Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-11T12:03:11.75177162Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:03:11.751791832Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.747µs success=true
level=info ts=2026-05-11T12:03:11.751868533Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:03:11.751878616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:11.751890445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:03:11.751900941Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:03:11.751978526Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=debug ts=2026-05-11T12:03:12.202166696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:03:12.202226363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:03:12.216762359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:03:15.127687517Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-11T12:03:16.752628911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:18.502619634Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=warn ts=2026-05-11T12:03:21.752769959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:21.877200597Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:03:25.252491894Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-11T12:03:25.368944204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:03:25.380064362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=warn ts=2026-05-11T12:03:26.752234422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:03:27.7218549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:03:27.72918916Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:03:28.628076732Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=debug ts=2026-05-11T12:03:28.628102096Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-11T12:03:29.081735826Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:03:31.752697515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:32.002281659Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-11T12:03:35.377916649Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-11T12:03:36.75262497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:38.009478907Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:03:38.753080873Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:03:39.736986682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="33 MB" avg_comp="2.7 MB" total_uncomp="266 MB" avg_uncomp="22 MB" full=11 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:03:39.746440809Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:03:41.752010214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:03:41.752062596Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:03:41.752076745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:03:41.752087509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:03:42.153996707Z caller=checkpoint.go:372 msg="writing series" size="35 MB"
level=debug ts=2026-05-11T12:03:42.15402448Z caller=checkpoint.go:384 msg="flushing series" totalSize="36 MB" series=4
level=debug ts=2026-05-11T12:03:45.50284666Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-11T12:03:46.363607363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:03:46.752536981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:48.877840897Z caller=checkpoint.go:372 msg="writing series" size="573 kB"
level=warn ts=2026-05-11T12:03:51.752783571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:03:52.252836143Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:03:52.252862891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=debug ts=2026-05-11T12:03:55.627632055Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-11T12:03:56.752350375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:03:58.882694434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:03:58.892656586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:03:59.002204872Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:03:59.082758741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:04:01.752288663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:04:01.821644086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:04:01.832063964Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:04:02.378085727Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-05-11T12:04:05.752648945Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-11T12:04:06.752184935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:09.128417949Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:04:09.128446944Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=5
level=info ts=2026-05-11T12:04:11.730204386Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:04:11.730220508Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:04:11.752232762Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:04:11.752260529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:04:11.752269012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:04:11.752278005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:12.503455106Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T12:04:15.165150019Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58764"
level=debug ts=2026-05-11T12:04:15.877732385Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:04:16.752890492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:18.394285274Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:04:19.252666512Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=debug ts=2026-05-11T12:04:21.513249303Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:04:21.752891443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:22.628080341Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=debug ts=2026-05-11T12:04:26.002714223Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=debug ts=2026-05-11T12:04:26.002735319Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-11T12:04:26.466473016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:04:26.475432203Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:04:26.752076108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:04:29.081360842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="36 MB" avg_comp="2.7 MB" total_uncomp="290 MB" avg_uncomp="22 MB" full=12 synced=1 labels="{stream=\"openshift-pipelines\"}"
ts=2026-05-11T12:04:29.084783693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=error ts=2026-05-11T12:04:29.09090563Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:04:29.377969387Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-11T12:04:31.752015887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:32.752475951Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T12:04:36.127107914Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=warn ts=2026-05-11T12:04:36.752779834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:39.502365411Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-11T12:04:41.752394201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:04:41.752407008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:41.752422979Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:04:41.752432065Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:04:42.182983587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45150"
level=debug ts=2026-05-11T12:04:42.87766907Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-11T12:04:44.91917057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:04:44.928810642Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:04:46.252145298Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-11T12:04:46.562134206Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:04:46.752077437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:49.627633931Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=warn ts=2026-05-11T12:04:51.75228392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:04:53.002404534Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T12:04:54.291388773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:04:54.300716196Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:04:56.377216019Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:04:56.752238134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:04:59.086079205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:04:59.75328322Z caller=checkpoint.go:372 msg="writing series" size="618 kB"
level=debug ts=2026-05-11T12:04:59.75330853Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=10
level=warn ts=2026-05-11T12:05:01.752114415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:03.127818905Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=debug ts=2026-05-11T12:05:06.502520604Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:05:06.75271447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:07.228789892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:05:07.230806472Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:05:09.878733043Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-05-11T12:05:11.720218921Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:05:11.730093965Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:05:11.731138101Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:05:11.752231753Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:05:11.752250273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:11.752265246Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:05:11.752273899Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:05:12.965210234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:05:12.970104781Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:05:13.252913453Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-11T12:05:15.27777371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49134"
level=debug ts=2026-05-11T12:05:16.627234993Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T12:05:16.752451258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:20.002528413Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T12:05:21.752017927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:23.37992426Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-11T12:05:23.379949867Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=7
level=info ts=2026-05-11T12:05:23.900291333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="313 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:05:23.912406727Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:05:26.752443357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:26.752462869Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:05:29.087213249Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:05:30.127115012Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-11T12:05:31.752620355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:33.503421038Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-05-11T12:05:35.209479096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:05:35.22534205Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=warn ts=2026-05-11T12:05:36.752643586Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:36.877257966Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-11T12:05:39.085900278Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:05:39.627186361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:05:39.634731324Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:05:40.252650971Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T12:05:41.752369752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:05:41.752398648Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:05:41.752406381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:05:41.752419268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:43.627777085Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-11T12:05:46.541490767Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:05:46.752783816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:47.003693499Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:05:50.378330681Z caller=checkpoint.go:372 msg="writing series" size="472 kB"
level=warn ts=2026-05-11T12:05:51.752808443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:53.752508696Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=debug ts=2026-05-11T12:05:53.752539129Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-11T12:05:56.752291385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:05:57.127630533Z caller=checkpoint.go:372 msg="writing series" size="514 B"
ts=2026-05-11T12:05:59.088865371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:06:00.50575833Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:06:01.752081166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:06:02.768352128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:06:02.774095426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:06:03.877665728Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-11T12:06:06.752488272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:07.25302627Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:06:10.272967089Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:06:10.492587983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="38 MB" avg_comp="2.7 MB" total_uncomp="313 MB" avg_uncomp="22 MB" full=13 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:06:10.503623341Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:06:10.627436357Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-11T12:06:11.730366749Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:06:11.73079918Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:06:11.752188264Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:06:11.752201347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:11.752221364Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:06:11.752229397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:06:14.003013403Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
ts=2026-05-11T12:06:15.280494437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43208"
level=warn ts=2026-05-11T12:06:16.752352722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:17.378838142Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:06:18.464181608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:06:18.472798759Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:06:20.752850798Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:06:21.752771975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:24.128057803Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=info ts=2026-05-11T12:06:26.524106437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:06:26.533661042Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:06:26.533692205Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:06:26.533720005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:06:26.542675862Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:06:26.752742553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:27.502851655Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:06:29.090429768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:06:30.878720005Z caller=checkpoint.go:372 msg="writing series" size="816 kB"
level=debug ts=2026-05-11T12:06:30.878746123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=11
level=warn ts=2026-05-11T12:06:31.752433428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:34.252036782Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-11T12:06:36.752265423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:37.048276292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:06:37.627335033Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=debug ts=2026-05-11T12:06:41.003274627Z caller=checkpoint.go:372 msg="writing series" size="451 kB"
level=info ts=2026-05-11T12:06:41.756889822Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:06:41.756922075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:06:41.756932722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:06:41.757001254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:42.546976162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:06:42.975233766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:06:42.9827635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:06:44.378369297Z caller=checkpoint.go:372 msg="writing series" size="622 kB"
level=debug ts=2026-05-11T12:06:44.378392264Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-11T12:06:45.171527775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58444"
ts=2026-05-11T12:06:45.282119006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33014"
level=warn ts=2026-05-11T12:06:46.752395986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:47.752973737Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-11T12:06:51.129682406Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:06:51.752309489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:54.502251205Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-11T12:06:54.90626156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:06:54.913874359Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:06:56.752461599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:06:57.878237353Z caller=checkpoint.go:372 msg="writing series" size="397 kB"
level=debug ts=2026-05-11T12:06:57.878264719Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-11T12:06:59.092025204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:06:59.182164133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.8 MB" total_uncomp="336 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:06:59.190074522Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:07:01.25289027Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-11T12:07:01.752744043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:03.72767981Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:07:04.579154522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:07:04.587289042Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:07:04.627523464Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T12:07:06.75266896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:08.007701512Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-11T12:07:08.007731968Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=3
level=info ts=2026-05-11T12:07:10.556909272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:07:10.567951108Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:07:11.377506325Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-05-11T12:07:11.730008657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:07:11.73109023Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:07:11.7524616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:07:11.752476603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:11.75249206Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:07:11.752500569Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:07:14.752438607Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-11T12:07:16.752649182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:18.127630394Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-05-11T12:07:19.765482824Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:07:21.502947952Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
level=warn ts=2026-05-11T12:07:21.752808649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:24.877962546Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:07:26.752426912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:28.252962928Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
ts=2026-05-11T12:07:29.093533307Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:07:31.628111308Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T12:07:31.628137803Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T12:07:31.752494953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:31.862961427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:07:35.010429843Z caller=checkpoint.go:372 msg="writing series" size="7.4 MB"
level=debug ts=2026-05-11T12:07:35.010463776Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.4 MB" series=1
level=warn ts=2026-05-11T12:07:36.752027085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:07:38.37814967Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-11T12:07:41.75233442Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=info ts=2026-05-11T12:07:41.752346816Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:07:41.752372155Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:07:41.752379763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:07:41.752390089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:07:41.93768456Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-11T12:07:41.939803434Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.187865602s
level=info ts=2026-05-11T12:07:46.1596514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="41 MB" avg_comp="2.8 MB" total_uncomp="336 MB" avg_uncomp="22 MB" full=14 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:07:46.171891286Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:07:46.752785738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:07:47.893836833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:07:47.917955015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:07:48.528838803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:07:48.533364606Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:07:50.81606341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:07:50.830525889Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-gfer\"}"
level=warn ts=2026-05-11T12:07:51.751866107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:07:56.752282698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:07:59.094716491Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:08:01.751874414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:04.073165308Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:08:06.17269388Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:08:06.752487805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:11.729576576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:08:11.730589725Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:08:11.752354157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:08:11.7523886Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:08:11.752398263Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:08:11.752397454Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-11T12:08:11.752445916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:08:11.752620458Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-11T12:08:11.752631347Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:08:11.752654385Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.111µs success=true
level=debug ts=2026-05-11T12:08:11.752870321Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-11T12:08:12.192683931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56258"
level=debug ts=2026-05-11T12:08:15.128711074Z caller=checkpoint.go:372 msg="writing series" size="71 kB"
ts=2026-05-11T12:08:15.176043206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43184"
level=warn ts=2026-05-11T12:08:16.752674315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:18.503250064Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=warn ts=2026-05-11T12:08:21.752779696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:21.878499256Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:08:25.253316807Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-11T12:08:26.752110522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:28.628272166Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
ts=2026-05-11T12:08:29.09593958Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:08:30.077278291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:08:30.08758591Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:08:30.087627474Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:08:30.087678623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:08:30.097164355Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:08:31.751892273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:08:31.871982504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:08:31.879176927Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:08:32.005376721Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-11T12:08:35.378975721Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-05-11T12:08:35.378998737Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-11T12:08:36.752389602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:08:37.725105508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:08:37.729754345Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:08:38.35066619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:08:38.359166285Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:08:38.753708457Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:08:39.97421601Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:08:41.752008341Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:08:41.752038262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:08:41.752029422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:08:41.752046779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:08:42.181267423Z caller=checkpoint.go:372 msg="writing series" size="45 MB"
level=debug ts=2026-05-11T12:08:42.181298153Z caller=checkpoint.go:384 msg="flushing series" totalSize="45 MB" series=2
ts=2026-05-11T12:08:42.193970849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:58084"
level=debug ts=2026-05-11T12:08:42.404489318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:08:42.40679409Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T12:08:44.716027412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:08:44.725163569Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:08:45.502990153Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-11T12:08:46.752730457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:48.878411999Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-11T12:08:51.752615973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:52.255307424Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-11T12:08:52.255332356Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=debug ts=2026-05-11T12:08:55.628302379Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-11T12:08:56.752585742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:08:59.003783939Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:08:59.098049659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:09:01.752762042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:02.378255715Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:09:05.753136435Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-05-11T12:09:06.75237245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:09.129771921Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-11T12:09:09.129795755Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=5
level=debug ts=2026-05-11T12:09:10.94442023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:09:11.730366252Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:09:11.731557656Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-11T12:09:11.752367957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:09:11.752382386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:09:11.752399816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:09:11.75240686Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:09:12.195718427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49092"
level=debug ts=2026-05-11T12:09:12.503605458Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-11T12:09:15.878493879Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:09:16.751941706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:19.253312812Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-11T12:09:21.752859397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:22.629372763Z caller=checkpoint.go:372 msg="writing series" size="444 kB"
level=debug ts=2026-05-11T12:09:22.629402524Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-11T12:09:22.67208186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:09:22.68090497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:09:23.899151592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:09:26.003824617Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
level=warn ts=2026-05-11T12:09:26.752197087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:09:27.001820257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:09:27.016644624Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T12:09:29.100709681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:09:29.378750636Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-11T12:09:31.279108449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:09:31.283659972Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:09:31.283700931Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:09:31.283736142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:09:31.287820952Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:09:31.752829173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:32.753648811Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T12:09:36.128989698Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=warn ts=2026-05-11T12:09:36.752656893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:09:38.251845869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:09:38.262008087Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:09:38.262034177Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:09:38.262061061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:09:38.270294325Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:09:39.503956093Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-11T12:09:41.752293722Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:09:41.752318512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:41.752332481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:09:41.752342343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:09:42.878895379Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-05-11T12:09:43.319446487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T12:09:45.180803106Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36048"
level=debug ts=2026-05-11T12:09:46.253485101Z caller=checkpoint.go:372 msg="writing series" size="402 kB"
level=warn ts=2026-05-11T12:09:46.752551442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:09:47.101225379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:09:47.10753357Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:09:49.629068495Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=debug ts=2026-05-11T12:09:49.629100124Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-11T12:09:51.466447278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:09:51.474642858Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=warn ts=2026-05-11T12:09:51.752896527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:09:53.002980096Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-11T12:09:56.378299051Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:09:56.752022291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:09:59.102639928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:09:59.753983437Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=warn ts=2026-05-11T12:10:01.752146245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:03.129424852Z caller=checkpoint.go:372 msg="writing series" size="821 kB"
level=debug ts=2026-05-11T12:10:03.129455368Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-11T12:10:06.503650481Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:10:06.752619462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:10:07.011248167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:10:07.020984154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:10:08.144623665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:10:08.153538817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:10:09.272735778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:10:09.28191055Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:10:09.878230842Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-05-11T12:10:10.843980134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:10:11.729612317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:10:11.730676619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:10:11.751818662Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:10:11.751855194Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:10:11.751868892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:10:11.751906265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:12.037964752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:10:12.199074456Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:50636"
level=debug ts=2026-05-11T12:10:13.255370573Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:10:13.255393961Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-11T12:10:15.929414763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:10:15.939883252Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:10:16.628110573Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T12:10:16.752629231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:20.003844241Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T12:10:21.752510283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:23.382456707Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-11T12:10:23.382484463Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=3
level=warn ts=2026-05-11T12:10:26.752356312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:26.75346507Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:10:29.104387062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:10:30.128676916Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=warn ts=2026-05-11T12:10:31.751980459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:33.503612269Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=debug ts=2026-05-11T12:10:35.09267511Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:10:35.092771392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:10:35.100905207Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:10:36.752330191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:36.878708099Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-11T12:10:40.253695985Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-05-11T12:10:41.75227009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:10:41.752300777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:10:41.752309337Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:10:41.752318832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:43.628956631Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
ts=2026-05-11T12:10:45.183211187Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43592"
level=warn ts=2026-05-11T12:10:46.752085368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:47.003556306Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:10:50.378766279Z caller=checkpoint.go:372 msg="writing series" size="507 kB"
level=warn ts=2026-05-11T12:10:51.752473093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:10:53.663281579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:10:53.669103072Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:10:53.753957588Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:10:53.753983926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-11T12:10:56.752625926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:10:57.128311824Z caller=checkpoint.go:372 msg="writing series" size="514 B"
level=info ts=2026-05-11T12:10:57.496554468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:10:57.506167167Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=info ts=2026-05-11T12:10:58.82625895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.3 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:10:58.834860334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T12:10:59.105844664Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:11:00.5040127Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-11T12:11:00.934794293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:11:01.752520064Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:11:03.577024295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:11:03.585399616Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:11:03.87837939Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-11T12:11:06.752623547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:07.25313309Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:11:07.261430213Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:11:08.431797436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:11:08.436152882Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:11:10.628165318Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-11T12:11:11.729992105Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:11:11.731120231Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:11:11.752024778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:11:11.752052509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:11:11.752060772Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:11:11.752046646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:11:12.202607177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:45720"
level=debug ts=2026-05-11T12:11:14.003484254Z caller=checkpoint.go:372 msg="writing series" size="147 kB"
ts=2026-05-11T12:11:15.297951824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:33362"
level=warn ts=2026-05-11T12:11:16.752758223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:17.378549092Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-11T12:11:20.753421048Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:11:21.751861028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:24.129183931Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:11:24.12921429Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-11T12:11:26.752351031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:27.503119341Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:11:29.107895067Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:11:30.401218115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:11:30.408875344Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:11:30.879186646Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-11T12:11:30.879211329Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-11T12:11:31.751923286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:34.253866706Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-11T12:11:36.546410926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:11:36.752845647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:37.628443782Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-11T12:11:40.721836385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:11:40.734974341Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:11:41.004434004Z caller=checkpoint.go:372 msg="writing series" size="723 kB"
level=info ts=2026-05-11T12:11:41.752841533Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:11:41.752855664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:41.752875937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:11:41.752886173Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:11:44.378978564Z caller=checkpoint.go:372 msg="writing series" size="795 kB"
level=debug ts=2026-05-11T12:11:44.37900299Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-11T12:11:44.533038813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:11:44.543360804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T12:11:45.185999256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47232"
ts=2026-05-11T12:11:45.298972288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59882"
level=warn ts=2026-05-11T12:11:46.751911299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:47.754380253Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:11:47.754403642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=1
level=debug ts=2026-05-11T12:11:48.241337217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:11:50.315419521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:11:50.325052451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:11:51.128743065Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:11:51.751901266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:54.502967406Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-11T12:11:54.900023798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:11:54.904714362Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:11:56.752644451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:11:57.87879851Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
ts=2026-05-11T12:11:59.109980441Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:12:01.253605075Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-11T12:12:01.752690138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:04.6289029Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T12:12:06.752867773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:07.479444384Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:12:08.006032405Z caller=checkpoint.go:372 msg="writing series" size="3.0 MB"
level=debug ts=2026-05-11T12:12:08.006063492Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=6
level=debug ts=2026-05-11T12:12:11.378906684Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-11T12:12:11.730785734Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:12:11.732145396Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:12:11.75263336Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:12:11.75267008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:12:11.752662519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:12:11.752678215Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:12:12.204734381Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33472"
level=info ts=2026-05-11T12:12:14.742634706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:12:14.747695551Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:12:14.753772262Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-11T12:12:16.75229493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:18.128855446Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=debug ts=2026-05-11T12:12:21.503881281Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-05-11T12:12:21.751921861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:12:24.280388667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:12:24.287929492Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:12:24.287957536Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:12:24.287984943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:12:24.295042218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:12:24.879020991Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:12:26.752666998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:28.253895815Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
ts=2026-05-11T12:12:29.111392678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:12:31.628163785Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T12:12:31.628191345Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T12:12:31.752433782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:35.008811876Z caller=checkpoint.go:372 msg="writing series" size="7.9 MB"
level=debug ts=2026-05-11T12:12:35.008838091Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.9 MB" series=1
level=debug ts=2026-05-11T12:12:36.477832015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:12:36.479605718Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T12:12:36.752863554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:38.378436271Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T12:12:38.566626273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:12:38.574800916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:12:38.875720998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:12:38.883908138Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:12:39.570126148Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:12:41.75232699Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:12:41.752356349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:12:41.75236466Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:12:41.752362174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:12:41.753418453Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=info ts=2026-05-11T12:12:41.763172552Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-11T12:12:41.765178683Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.012363764s
level=info ts=2026-05-11T12:12:42.235268606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="44 MB" avg_comp="2.8 MB" total_uncomp="360 MB" avg_uncomp="22 MB" full=15 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:12:42.244336298Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:12:46.752593767Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:12:49.137191252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:12:49.142324535Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:12:51.753111497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:12:56.752624942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:12:59.112907283Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:13:01.752326408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:13:06.752841372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:13:07.878562718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:13:07.885174018Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:13:07.983041166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:13:09.659306024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:13:09.667491235Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:13:11.729979129Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:13:11.731031026Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:13:11.752321747Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:13:11.752372329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:13:11.75237207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:11.752396752Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:13:11.752405572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:13:11.752523763Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-11T12:13:11.752532472Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:13:11.752551012Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.366µs success=true
level=debug ts=2026-05-11T12:13:11.752752567Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=debug ts=2026-05-11T12:13:15.128389209Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-11T12:13:16.752393401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:18.504094822Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=warn ts=2026-05-11T12:13:21.752440741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:21.878948057Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:13:25.253384938Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-11T12:13:26.752905962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:28.628016435Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
ts=2026-05-11T12:13:29.114798371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:13:31.752127687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:32.003346625Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-11T12:13:32.003371098Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-11T12:13:32.779412882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:13:32.788934848Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:13:35.37835606Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-05-11T12:13:35.554936034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:13:35.562010958Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:13:36.090912462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:13:36.101488248Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:13:36.752214152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:38.753265234Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:13:41.752381055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:13:41.752387226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:41.75241031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:13:41.752420834Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:13:42.176889794Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-11T12:13:42.176919645Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=3
ts=2026-05-11T12:13:42.209529954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39264"
level=info ts=2026-05-11T12:13:42.313479824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:13:42.321138314Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:13:45.503007259Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-11T12:13:46.752087034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:48.878182269Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=info ts=2026-05-11T12:13:51.64391358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:13:51.654229734Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:13:51.752411309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:52.256008683Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-11T12:13:52.256033323Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=3
level=debug ts=2026-05-11T12:13:55.62806092Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-11T12:13:56.7526253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:13:59.00375011Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:13:59.116397913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:13:59.259059704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:13:59.264406386Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:14:01.75234182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:02.378924641Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:14:03.215630582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:14:05.753732777Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-11T12:14:06.752396128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:09.131017386Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-11T12:14:09.131052366Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=5
level=debug ts=2026-05-11T12:14:11.730015038Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:14:11.731080295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:14:11.75268485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:14:11.752715065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:14:11.752723007Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:14:11.752820613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:12.398960829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:14:12.503929486Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T12:14:15.306693085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39230"
level=info ts=2026-05-11T12:14:15.672383185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:14:15.680981766Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:14:15.877820558Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:14:16.752948401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:14:18.13871634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:14:18.146696203Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:14:19.253228843Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-11T12:14:21.752836508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:22.628654824Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:14:22.628678728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:14:26.003486195Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=warn ts=2026-05-11T12:14:26.75282996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:14:29.118720608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:14:29.378749416Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-11T12:14:31.363852652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:14:31.373031853Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:14:31.752134181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:32.75306808Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T12:14:34.133493419Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:14:36.128560865Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=warn ts=2026-05-11T12:14:36.752275784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:39.503624018Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=warn ts=2026-05-11T12:14:41.752374076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:14:41.752363187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:14:41.752394592Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:14:41.752406995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:14:42.878755005Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
ts=2026-05-11T12:14:45.193951335Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39518"
ts=2026-05-11T12:14:45.308453209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55734"
level=debug ts=2026-05-11T12:14:46.253478856Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=debug ts=2026-05-11T12:14:46.253503267Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T12:14:46.752812812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:14:47.289990736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:14:47.30105145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:14:49.628432184Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-11T12:14:49.854335215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:14:49.860195147Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:14:51.752766093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:53.002853626Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-11T12:14:56.377965861Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:14:56.752203698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:14:58.783782801Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:14:59.120076753Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:14:59.211977208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:14:59.221073244Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=debug ts=2026-05-11T12:14:59.754107664Z caller=checkpoint.go:372 msg="writing series" size="530 kB"
level=warn ts=2026-05-11T12:15:01.752841028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:03.129188837Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-05-11T12:15:03.129212066Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-05-11T12:15:06.503665914Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:15:06.752609977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:09.248657561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:15:09.878620789Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-11T12:15:11.730593598Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:15:11.731305557Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T12:15:11.731361191Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976112 n=1
level=debug ts=2026-05-11T12:15:11.731411773Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976112 n=1
level=debug ts=2026-05-11T12:15:11.731411139Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:15:11.752212887Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:15:11.75224085Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:15:11.75225197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:15:11.752252307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:13.25488069Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:15:13.254926453Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
ts=2026-05-11T12:15:15.196002627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59656"
level=debug ts=2026-05-11T12:15:16.628156783Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-11T12:15:16.752752557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:15:16.991292331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:15:17.00433352Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:15:17.004370835Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:15:17.004423631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:15:17.015216519Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:15:19.496526066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:15:19.502159302Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:15:20.003725756Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T12:15:21.752774483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:23.385122665Z caller=checkpoint.go:372 msg="writing series" size="7.8 MB"
level=debug ts=2026-05-11T12:15:23.385146907Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=3
level=warn ts=2026-05-11T12:15:26.752328764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:26.753402162Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:15:29.122006616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:15:30.128613223Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-05-11T12:15:31.752841424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:33.503757236Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-05-11T12:15:34.555996548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:15:34.570250141Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:15:36.184394057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:15:36.1903288Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:15:36.752774243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:36.878287552Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-11T12:15:37.702790315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:15:40.253590383Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-11T12:15:41.75247806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:15:41.752508642Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:15:41.752518405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:15:41.75255083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:15:43.55024768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5cd6e63403151a15 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:15:43.559209815Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5cd6e63403151a15/19e16de1580%3A19e16eb5fff%3Ad2d2e486?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-gfer\"}"
level=error ts=2026-05-11T12:15:43.559243731Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:15:43.559283923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:15:43.567466214Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:15:43.62851211Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-11T12:15:45.028791659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:15:45.036046396Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:15:46.752847916Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:47.003419397Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:15:50.320463598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:15:50.378882717Z caller=checkpoint.go:372 msg="writing series" size="601 kB"
level=debug ts=2026-05-11T12:15:50.378906777Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-11T12:15:51.751920344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:53.7536947Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-11T12:15:56.752146487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:15:57.128252656Z caller=checkpoint.go:372 msg="writing series" size="599 B"
ts=2026-05-11T12:15:59.124018912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:16:00.50347289Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-11T12:16:01.217816846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:16:01.229163Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:16:01.752311773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:16:02.53601135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:16:02.545217589Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:16:03.87861484Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-11T12:16:04.096351489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:16:06.752426847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:07.253408304Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:16:10.62847008Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=debug ts=2026-05-11T12:16:11.729526343Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:16:11.730592147Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-11T12:16:11.752463702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:16:11.752458986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:16:11.752487131Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:16:11.752495122Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:16:14.003416493Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-11T12:16:15.200032875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45826"
level=warn ts=2026-05-11T12:16:16.752298977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:17.378821852Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:16:18.929508612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:16:18.934713956Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:16:18.934741303Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:16:18.934768978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:16:18.939059577Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:16:20.753840162Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:16:21.751872261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:24.128985015Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:16:24.129011774Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:16:26.744321277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:16:26.751914445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-11T12:16:26.752314998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:16:27.503558476Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:16:29.125113339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:16:30.604275233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:16:30.609386225Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:16:30.878966513Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-11T12:16:30.878989504Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-11T12:16:31.495089516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:16:31.503116643Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:16:31.75208552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:34.253719679Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:16:34.677880626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:16:34.682833825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:16:36.752711019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:37.627876374Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-11T12:16:39.197510782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:16:41.003641998Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-11T12:16:41.268141553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:16:41.275866632Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:16:41.752049374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:16:41.752055343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:16:41.752083124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:16:41.752090793Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:16:42.218592055Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49294"
level=debug ts=2026-05-11T12:16:44.379175703Z caller=checkpoint.go:372 msg="writing series" size="735 kB"
level=debug ts=2026-05-11T12:16:44.379201261Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
ts=2026-05-11T12:16:45.201464681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53218"
level=debug ts=2026-05-11T12:16:46.512705577Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:16:46.514486338Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T12:16:46.752484654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:47.753662894Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-11T12:16:47.753688138Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=debug ts=2026-05-11T12:16:51.128791146Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:16:51.752849473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:54.503467822Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T12:16:56.752431774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:16:57.87904818Z caller=checkpoint.go:372 msg="writing series" size="430 kB"
ts=2026-05-11T12:16:59.126926659Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:17:01.253660913Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=warn ts=2026-05-11T12:17:01.752409617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:17:03.369220503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:17:03.375655811Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:17:04.628297274Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T12:17:06.752237362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:17:08.005552367Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-11T12:17:08.005597411Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=debug ts=2026-05-11T12:17:09.490783828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:17:11.377947871Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-11T12:17:11.730134345Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:17:11.73113597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:17:11.752116581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:17:11.7521347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:17:11.752148798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:17:11.752157806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:17:12.219850985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:55116"
level=debug ts=2026-05-11T12:17:14.753805881Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-11T12:17:16.752240836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:17:17.755386762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:17:17.765443602Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:17:18.128681539Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-11T12:17:20.226121934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:17:20.235176824Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:17:20.634634532Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:17:21.503307039Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=warn ts=2026-05-11T12:17:21.752532167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:17:22.634080453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:17:22.63887942Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:17:22.638906694Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:17:22.63893483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:17:22.643101721Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:17:24.878761698Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:17:26.752371369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:17:28.253232126Z caller=checkpoint.go:372 msg="writing series" size="454 kB"
level=debug ts=2026-05-11T12:17:28.253256878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-11T12:17:29.128032823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:17:31.628460038Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-11T12:17:31.751943152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:17:33.822445276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:17:33.830694169Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:17:34.780194639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:17:35.009710921Z caller=checkpoint.go:372 msg="writing series" size="8.4 MB"
level=debug ts=2026-05-11T12:17:35.009735336Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.6 MB" series=2
level=warn ts=2026-05-11T12:17:36.752723723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:17:37.545932153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="47 MB" avg_comp="2.8 MB" total_uncomp="384 MB" avg_uncomp="23 MB" full=16 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:17:37.554018916Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:17:38.378937158Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T12:17:41.752169706Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:17:41.752196873Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:17:41.752206248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:17:41.75221922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:17:41.753264191Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=info ts=2026-05-11T12:17:41.760017517Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-11T12:17:41.761418184Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00870895s
ts=2026-05-11T12:17:42.221102818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:43730"
ts=2026-05-11T12:17:45.318682034Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35288"
level=info ts=2026-05-11T12:17:45.607851436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:17:45.61291246Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:17:46.752930687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:17:51.752430428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:17:56.752352001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:17:59.129478252Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:18:01.585360648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:18:01.751874053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:18:02.360953586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:18:02.369488794Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:18:04.101873194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:18:04.107363609Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=info ts=2026-05-11T12:18:06.507046948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:18:06.515158259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:18:06.678075701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:18:06.752302321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:11.729695812Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:18:11.73076292Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:18:11.751775844Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:18:11.751796105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:18:11.751821482Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:18:11.751830396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:18:11.751864948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:18:11.75193982Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-11T12:18:11.75194946Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:18:11.75196665Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.822µs success=true
level=debug ts=2026-05-11T12:18:11.752173718Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
ts=2026-05-11T12:18:12.222315845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:56564"
level=debug ts=2026-05-11T12:18:15.127808105Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
ts=2026-05-11T12:18:15.20673064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54692"
level=warn ts=2026-05-11T12:18:16.752410682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:18.503001293Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:18:20.88991537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="50 MB" avg_comp="2.8 MB" total_uncomp="408 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:18:20.901335351Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:18:21.752336872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:21.878002517Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:18:25.253035838Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-11T12:18:26.751893808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:28.627694695Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-11T12:18:28.627717664Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-11T12:18:29.130472189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:18:31.752117437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:32.00279821Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-11T12:18:35.377937739Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
level=warn ts=2026-05-11T12:18:36.752634566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:38.569510444Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:18:38.752670759Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:18:41.752463645Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:18:41.75249587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:18:41.752506549Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:18:41.752513576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:42.17751981Z caller=checkpoint.go:372 msg="writing series" size="51 MB"
level=debug ts=2026-05-11T12:18:42.177557015Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=4
ts=2026-05-11T12:18:45.207961648Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39454"
level=info ts=2026-05-11T12:18:45.317034161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
ts=2026-05-11T12:18:45.321033822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39906"
level=error ts=2026-05-11T12:18:45.321854257Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:18:45.503076478Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-05-11T12:18:45.517168913Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:18:46.753066591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:18:47.375628289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:18:47.383733737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:18:47.383761978Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:18:47.383789524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:18:47.391436841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:18:48.878219259Z caller=checkpoint.go:372 msg="writing series" size="782 kB"
level=info ts=2026-05-11T12:18:51.40793123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:18:51.418859198Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:18:51.752085199Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:18:52.254791183Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-11T12:18:52.25481754Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=debug ts=2026-05-11T12:18:55.628232925Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-11T12:18:56.75240542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:18:58.324620883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:18:58.330059587Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:18:59.002925058Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:18:59.131888937Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:19:01.752503516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:02.378484116Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:19:04.746795221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:19:04.756555548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:19:05.752735234Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-05-11T12:19:06.751931338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:09.129797539Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-11T12:19:09.129820884Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=debug ts=2026-05-11T12:19:09.224877548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:19:11.730310856Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:19:11.730336408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:19:11.752265741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:19:11.752298948Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:19:11.752307488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:19:11.752347529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:19:12.225716959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35850"
level=debug ts=2026-05-11T12:19:12.503583084Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T12:19:15.322763419Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47486"
level=info ts=2026-05-11T12:19:15.804704444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="50 MB" avg_comp="2.8 MB" total_uncomp="408 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:19:15.816974782Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:19:15.878074654Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:19:16.752622411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:19.252941447Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-11T12:19:21.752081051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:22.628245316Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:19:22.628283053Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:19:26.003137009Z caller=checkpoint.go:372 msg="writing series" size="345 kB"
level=warn ts=2026-05-11T12:19:26.752479842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:19:29.133546778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:19:29.378019918Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-11T12:19:31.752661863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:32.752429621Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T12:19:34.474635048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:19:34.474694726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="343 kB" avg_comp="343 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:19:34.480349486Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:19:34.99896158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:19:35.007159014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:19:36.127994062Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T12:19:36.752632687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:19:38.966221925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:19:38.974860544Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:19:39.503328367Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:19:41.752028233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:19:41.752058904Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:19:41.75204601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:19:41.752066916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:19:42.878090548Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=info ts=2026-05-11T12:19:44.611041747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:19:44.616655935Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=info ts=2026-05-11T12:19:44.829466366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:19:44.834040014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-11T12:19:45.324686316Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37326"
level=debug ts=2026-05-11T12:19:46.252768453Z caller=checkpoint.go:372 msg="writing series" size="433 kB"
level=debug ts=2026-05-11T12:19:46.252794782Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-05-11T12:19:46.506933944Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:19:46.752046728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:19:46.84861871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:19:46.853039951Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:19:49.628236797Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=warn ts=2026-05-11T12:19:51.752812726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:19:53.003003321Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-11T12:19:56.377513536Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:19:56.752408683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:19:57.421645278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="50 MB" avg_comp="2.8 MB" total_uncomp="408 MB" avg_uncomp="23 MB" full=17 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:19:57.433325609Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T12:19:59.135067991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:19:59.753239934Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-11T12:20:01.752576085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:01.828953073Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:20:03.128349054Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-05-11T12:20:03.128375039Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-11T12:20:06.503175222Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:20:06.752289208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:09.877849388Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-11T12:20:11.730209571Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:20:11.730833395Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:20:11.752470903Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:20:11.752491844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:11.752503032Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:20:11.752512603Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:20:13.253478607Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:20:13.253501713Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
ts=2026-05-11T12:20:15.21232722Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36558"
level=debug ts=2026-05-11T12:20:16.628289896Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-11T12:20:16.752191628Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:20.002488469Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:20:21.679791886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:20:21.68831183Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:20:21.752061602Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:20:22.395645216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:20:22.404523003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:20:23.387390469Z caller=checkpoint.go:372 msg="writing series" size="9.3 MB"
level=debug ts=2026-05-11T12:20:23.387414553Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.3 MB" series=3
level=info ts=2026-05-11T12:20:25.465622406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:20:25.471214295Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:20:26.752553033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:26.752559637Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-11T12:20:28.66244831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:20:28.667853693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-11T12:20:29.136936782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:20:30.127821143Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-11T12:20:31.504808282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:20:31.509889891Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:20:31.752710766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:32.221432977Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:20:33.50304706Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T12:20:36.752898339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:36.877907514Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-11T12:20:40.253098936Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=warn ts=2026-05-11T12:20:41.752223459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:20:41.752209783Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:20:41.752247059Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:20:41.752258349Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:20:43.627913145Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
ts=2026-05-11T12:20:45.327836274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49876"
level=debug ts=2026-05-11T12:20:46.488544223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:20:46.490314169Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T12:20:46.752773342Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:47.002394452Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:20:50.37831738Z caller=checkpoint.go:372 msg="writing series" size="576 kB"
level=debug ts=2026-05-11T12:20:50.3783526Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-11T12:20:51.698743262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="53 MB" avg_comp="2.8 MB" total_uncomp="433 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:20:51.709231013Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:20:51.752515216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:53.752513314Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-11T12:20:56.752113855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:20:57.127725008Z caller=checkpoint.go:372 msg="writing series" size="641 B"
ts=2026-05-11T12:20:59.138823432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:21:00.502636763Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:21:01.751974484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:03.878047748Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=debug ts=2026-05-11T12:21:04.848178566Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:21:06.752469093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:07.25246304Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:21:10.627920262Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-11T12:21:11.322179204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:21:11.328045574Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:21:11.730375914Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:21:11.730394319Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:21:11.752168714Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:21:11.75219638Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:21:11.752204998Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:21:11.752213834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:14.002723252Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T12:21:14.418850074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:21:14.426354774Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:21:15.201703715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:21:15.207376687Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
ts=2026-05-11T12:21:15.215775571Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55702"
level=info ts=2026-05-11T12:21:15.610182844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:21:15.615023024Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:21:16.752052896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:17.378066936Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-11T12:21:20.752733491Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-11T12:21:21.154516369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:21:21.163706488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:21:21.752617877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:21.929047543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:21:24.127755007Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:21:24.127780682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-05-11T12:21:26.752338933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:27.502717792Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:21:29.140836955Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:21:30.878897128Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=warn ts=2026-05-11T12:21:31.752544902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:33.052513329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:21:34.252228541Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:21:34.338676348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="53 MB" avg_comp="2.8 MB" total_uncomp="433 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:21:34.350221344Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:21:34.350248094Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:21:34.350292371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="53 MB" avg_comp="2.8 MB" total_uncomp="433 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:21:34.357909446Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:21:36.755427738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:37.627938318Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-11T12:21:37.627963312Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-11T12:21:41.002436319Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=info ts=2026-05-11T12:21:41.752033351Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:21:41.752067282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:21:41.752054664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:21:41.752075432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:21:44.378125227Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:21:44.37814944Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
ts=2026-05-11T12:21:45.217442713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48594"
ts=2026-05-11T12:21:45.331114083Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39442"
level=warn ts=2026-05-11T12:21:46.752631407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:47.754335442Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-11T12:21:47.754361471Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=info ts=2026-05-11T12:21:50.076472237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="53 MB" avg_comp="2.8 MB" total_uncomp="433 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:21:50.087269934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:21:51.12754501Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:21:51.752857115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:21:54.502281647Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T12:21:56.752058922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:21:57.527526878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:21:57.535102218Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:21:57.878410778Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
ts=2026-05-11T12:21:59.142716281Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:22:01.252714066Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=warn ts=2026-05-11T12:22:01.752408521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:22:02.090895759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:22:02.099423456Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:22:02.099455995Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:22:02.099489443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:22:02.108764764Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:22:04.627921669Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-11T12:22:06.752490626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:07.851645733Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:22:07.851718354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="454 kB" avg_comp="454 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:22:07.857127286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:22:08.006861259Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-11T12:22:08.006886126Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=6
level=info ts=2026-05-11T12:22:08.770401383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:22:08.775105176Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:22:11.155112502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="53 MB" avg_comp="2.8 MB" total_uncomp="433 MB" avg_uncomp="23 MB" full=18 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:22:11.164850427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:22:11.377797595Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-11T12:22:11.729704868Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:22:11.730759228Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:22:11.751828455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:22:11.751860968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:22:11.751869236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:22:11.751864034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:22:12.178483236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:22:12.184674449Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:22:14.526323155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:22:14.532066998Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:22:14.752803819Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-11T12:22:15.218835273Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44662"
ts=2026-05-11T12:22:15.332852702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50416"
level=info ts=2026-05-11T12:22:15.911045123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:22:15.921214719Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:22:16.752753673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:17.330917424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:22:18.128328003Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-11T12:22:21.50249935Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=warn ts=2026-05-11T12:22:21.752228682Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:22:22.11763059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:22:22.12316913Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:22:24.878142113Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:22:26.752332013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:28.252945926Z caller=checkpoint.go:372 msg="writing series" size="349 kB"
ts=2026-05-11T12:22:29.143762967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:22:31.628617824Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T12:22:31.628644111Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:22:31.752433094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:34.165443259Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:22:35.01062835Z caller=checkpoint.go:372 msg="writing series" size="9.2 MB"
level=debug ts=2026-05-11T12:22:35.01065395Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=1
level=warn ts=2026-05-11T12:22:36.752539714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:38.377902906Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T12:22:41.75216109Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:22:41.752157718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:22:41.75219197Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-05-11T12:22:41.752188345Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=info ts=2026-05-11T12:22:41.752203257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:22:41.759885139Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-11T12:22:41.761464873Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00933782s
level=warn ts=2026-05-11T12:22:46.752622965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:22:51.752707686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:22:52.863207255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:22:52.87327116Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:22:54.94054846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:22:54.945797372Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:22:55.022990604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:22:55.03146523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:22:56.752342164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:22:57.085471252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:22:57.089706448Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:22:59.145145525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:22:59.309170761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:22:59.314097115Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:22:59.314126387Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:22:59.314153678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:22:59.318168671Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:23:01.597126455Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:23:01.75282578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:23:04.538845371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:23:04.543483179Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:23:06.752643798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:23:08.897652691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:23:08.904916964Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:23:10.296174781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:23:10.30112584Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:23:11.730325858Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:23:11.730337103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:23:11.751432002Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:23:11.751653684Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-11T12:23:11.751665835Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:23:11.751692037Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.999µs success=true
level=info ts=2026-05-11T12:23:11.75186254Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:23:11.751887658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:23:11.751897581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:23:11.751901863Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=warn ts=2026-05-11T12:23:11.751905622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:15.127846673Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-11T12:23:16.752054553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:18.502832576Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-11T12:23:19.176690682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:23:21.752818363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:21.877249355Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:23:25.252632082Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-11T12:23:26.752339132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:28.166783835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:23:28.627306686Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-11T12:23:28.627331117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-11T12:23:29.146836414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:23:31.752339698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:32.002537471Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-11T12:23:35.37778863Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-11T12:23:36.752903942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:38.752067045Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:23:41.752596632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:23:41.752628056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:23:41.752636936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:23:41.752634319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:42.194304255Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-11T12:23:42.194340038Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=4
level=info ts=2026-05-11T12:23:44.50830562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:23:44.517964413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T12:23:45.223242927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33680"
level=debug ts=2026-05-11T12:23:45.502528165Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-11T12:23:46.752247411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:23:47.849333249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:23:47.858188302Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:23:48.264408185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:23:48.269010536Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:23:48.879867874Z caller=checkpoint.go:372 msg="writing series" size="845 kB"
level=info ts=2026-05-11T12:23:50.14314023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:23:50.148281623Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:23:51.752205117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:52.253673266Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-11T12:23:52.253698067Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.0 MB" series=3
level=info ts=2026-05-11T12:23:53.343412264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:23:53.348559376Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:23:55.62753048Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=info ts=2026-05-11T12:23:56.115649155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:23:56.125064407Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:23:56.752493034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:23:59.002454185Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:23:59.148531855Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:23:59.780250107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:24:01.752657746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:02.37752449Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:24:02.535978547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:24:02.541624142Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:24:02.541649617Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:24:02.541677506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:24:02.546368304Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:24:05.752333374Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-11T12:24:06.752587299Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:09.130870265Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-11T12:24:09.130925297Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-05-11T12:24:11.730374833Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:24:11.730388352Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:24:11.752671732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:24:11.752709346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:24:11.752717775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:24:11.752818722Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:24:12.242461293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57822"
level=debug ts=2026-05-11T12:24:12.502604849Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-11T12:24:13.797795227Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:24:15.877240006Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:24:16.752710338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:24:19.206601704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:24:19.212165966Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:24:19.252427881Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-11T12:24:21.752708488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:22.628279469Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:24:22.628301624Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:24:26.002211322Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=warn ts=2026-05-11T12:24:26.75265202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:24:29.150641501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:24:29.377286157Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-11T12:24:29.502332281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:24:29.511885916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:24:29.613291288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:24:29.618117603Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:24:31.163870614Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:24:31.752855525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:32.75226759Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-11T12:24:34.666161779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:24:34.672974767Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:24:36.127734699Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-11T12:24:36.199859288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:24:36.207482773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:24:36.752693846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:39.502267463Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-05-11T12:24:41.752833671Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:24:41.752851774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:41.752866761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:24:41.752876047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:24:42.116416123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:24:42.125545117Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:24:42.877771147Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=debug ts=2026-05-11T12:24:46.253223082Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=warn ts=2026-05-11T12:24:46.752332755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:49.627386988Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=debug ts=2026-05-11T12:24:49.627411485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-11T12:24:51.752554446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:24:53.002163343Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T12:24:53.11252968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:24:53.119774319Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:24:53.376629769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:24:53.382091581Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:24:56.37706725Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:24:56.752890989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:24:59.152250908Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:24:59.75321586Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=warn ts=2026-05-11T12:25:01.75186622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:03.128094155Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-05-11T12:25:03.128120683Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=debug ts=2026-05-11T12:25:04.716015905Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:25:06.502762513Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:25:06.75219153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:09.877354009Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-11T12:25:11.730680805Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:25:11.731233036Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:25:11.752662729Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:25:11.752695765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:25:11.752706578Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:25:11.752754249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:13.253920883Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:25:13.253943459Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-11T12:25:16.189737794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:25:16.191895451Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:25:16.627305845Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-11T12:25:16.752411926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:25:19.085176172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:25:19.098336531Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:25:19.09836326Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:25:19.098391122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:25:19.105773535Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:25:20.002511973Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:25:21.437067893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:25:21.444952523Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:25:21.752847151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:23.387167538Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-11T12:25:23.387205621Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
level=info ts=2026-05-11T12:25:24.594133511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:25:24.599351514Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:25:26.752428729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:26.752433386Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:25:29.153536801Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:25:29.460092114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:25:29.465298481Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:25:30.127658854Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-05-11T12:25:30.580201482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:25:30.589060017Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:25:31.751886236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:33.502620705Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T12:25:36.752834146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:25:36.83719438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:25:36.845486576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:25:36.877578369Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-11T12:25:40.252934121Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-11T12:25:40.349367257Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:25:40.355431818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:25:40.359974235Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:25:40.431197317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:25:40.436452657Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:25:41.752007686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:25:41.752013212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:25:41.752041511Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:25:41.752050915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:25:43.627427496Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-11T12:25:45.23013289Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59384"
level=warn ts=2026-05-11T12:25:46.752168671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:47.002413855Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:25:50.377825937Z caller=checkpoint.go:372 msg="writing series" size="784 kB"
level=debug ts=2026-05-11T12:25:50.377855882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-11T12:25:51.752202077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:53.752257044Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=info ts=2026-05-11T12:25:53.968701218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:25:53.979209791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:25:56.752347041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:25:57.127521773Z caller=checkpoint.go:372 msg="writing series" size="641 B"
ts=2026-05-11T12:25:59.155030175Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:25:59.590418395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:26:00.502584919Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:26:01.752262128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:03.87700036Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-11T12:26:06.752217493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:07.252532845Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:26:09.234492123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:26:10.627258611Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-11T12:26:11.343509383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:26:11.349173005Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:26:11.729792904Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:26:11.730867206Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:26:11.752288473Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:26:11.752299703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:11.752329282Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:26:11.752340338Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:26:14.002216112Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-11T12:26:16.752246928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:17.377809923Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:26:17.994633454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:26:18.006992803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:26:20.752881916Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:26:21.752327691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:24.128012361Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:26:24.128040735Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:26:24.664874115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:26:24.671093139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:26:26.752357921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:27.502522364Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-11T12:26:27.551991289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:26:27.556690925Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:26:27.556717988Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:26:27.556743765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:26:27.560554956Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:26:29.15650351Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:26:29.481000597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:26:29.488606733Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:26:30.878786381Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:26:30.878809816Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-11T12:26:31.752497887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:34.252645062Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:26:34.585640524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:26:34.592673463Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:26:36.752149378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:26:36.781293462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:26:36.785935093Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:26:37.62720306Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-05-11T12:26:38.360537821Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:26:41.002953238Z caller=checkpoint.go:372 msg="writing series" size="554 kB"
level=info ts=2026-05-11T12:26:41.752795379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:26:41.752818221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:41.752842164Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:26:41.75285229Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:26:43.698553669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:26:43.70334059Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:26:44.377846012Z caller=checkpoint.go:372 msg="writing series" size="844 kB"
level=debug ts=2026-05-11T12:26:44.377876257Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=4
level=warn ts=2026-05-11T12:26:46.752255216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:47.753447754Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-11T12:26:47.753472978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=1
level=debug ts=2026-05-11T12:26:51.127530816Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:26:51.752781379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:53.651619451Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:26:54.502738216Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T12:26:56.75269041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:26:57.877724108Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
ts=2026-05-11T12:26:59.157523048Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:27:01.252785534Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-11T12:27:01.752507483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:27:04.627555264Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=info ts=2026-05-11T12:27:04.855160381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:27:04.86159027Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:27:06.535795159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:27:06.540896697Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:27:06.752697665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:27:07.421226694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:27:07.426223418Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:27:08.006096298Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-05-11T12:27:08.006127136Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=6
level=info ts=2026-05-11T12:27:10.940666398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:27:10.949627683Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:27:11.377922366Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-11T12:27:11.730207847Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:27:11.730428594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:27:11.752398297Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:27:11.752435502Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:27:11.752423478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:27:11.752445402Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:27:12.251417588Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52608"
level=debug ts=2026-05-11T12:27:12.409405978Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:27:14.751985727Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
ts=2026-05-11T12:27:15.348034326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48386"
level=info ts=2026-05-11T12:27:15.388487595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="56 MB" avg_comp="2.8 MB" total_uncomp="458 MB" avg_uncomp="23 MB" full=19 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:27:15.397899036Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:27:16.752166814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:27:18.087887285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:27:18.096376325Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:27:18.127773007Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-11T12:27:20.643541172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:27:20.648287182Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:27:21.502904833Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-11T12:27:21.752028406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:27:24.877136693Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:27:26.752583066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:27:28.252607311Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
level=debug ts=2026-05-11T12:27:28.25263346Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
ts=2026-05-11T12:27:29.159518446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:27:31.627919201Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-11T12:27:31.752124115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:27:35.014748704Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-11T12:27:35.014775406Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=2
level=debug ts=2026-05-11T12:27:36.098455861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:27:36.752412057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:27:38.377499027Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T12:27:41.751924977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:27:41.751962968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:27:41.751971891Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:27:41.75194726Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=warn ts=2026-05-11T12:27:41.75200937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:27:41.758550418Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-11T12:27:41.760952555Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009102199s
level=info ts=2026-05-11T12:27:43.967146578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:27:43.967195293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-11T12:27:43.976272728Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:27:46.752727292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:27:51.752238473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:27:53.11616512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:27:53.121829794Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:27:56.69385333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:27:56.698546438Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:27:56.752284014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:27:59.16095304Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:27:59.787152322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:27:59.79519897Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:28:00.103908865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:28:00.113188942Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:28:00.415331107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:28:00.419930865Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:28:01.752230773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:05.762133867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:28:05.772343384Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:28:06.752658572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:10.536528404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:28:10.545088799Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:28:10.545114263Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:28:10.545165085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:28:10.553944986Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:28:11.102055519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:28:11.730024067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:28:11.731078541Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:28:11.752161095Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:28:11.752191737Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:28:11.752221318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:28:11.75223367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:28:11.752279364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:11.752362216Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-11T12:28:11.752373152Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:28:11.752395944Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.812µs success=true
level=debug ts=2026-05-11T12:28:11.752657582Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=debug ts=2026-05-11T12:28:13.949598876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:28:15.128104255Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-11T12:28:16.752133885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:18.342785703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:28:18.348606125Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:28:18.503702909Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=warn ts=2026-05-11T12:28:21.752480312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:21.878140237Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:28:25.253674356Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-05-11T12:28:26.752082128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:27.551174976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:28:27.562007828Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:28:27.660271808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:28:27.668431514Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:28:28.628192645Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=debug ts=2026-05-11T12:28:28.628217978Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-05-11T12:28:29.163043271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:28:31.75241071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:32.003530535Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-11T12:28:35.378972723Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=warn ts=2026-05-11T12:28:36.752631581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:28:38.215961973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:28:38.223844795Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:28:38.752944284Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:28:38.827038786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:28:38.832351072Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:28:38.832391123Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:28:38.832425843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:28:38.836959012Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:28:41.752271293Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:28:41.752287638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:41.752302143Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:28:41.752310406Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:28:42.195549062Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-11T12:28:42.195606481Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=4
level=debug ts=2026-05-11T12:28:45.503211745Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-11T12:28:46.752501031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:47.701007375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:28:48.879444052Z caller=checkpoint.go:372 msg="writing series" size="719 kB"
level=info ts=2026-05-11T12:28:50.485526609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:28:50.498302486Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:28:51.752629492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:52.257263713Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-11T12:28:52.257300191Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=3
level=info ts=2026-05-11T12:28:53.803906773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:28:53.814601946Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:28:53.814638174Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:28:53.81467656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:28:53.82426337Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:28:54.30094077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:28:54.307373076Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:28:54.3279665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:28:54.332299035Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:28:55.201675929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:28:55.213189529Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:28:55.627901226Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-11T12:28:56.752252574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:28:59.003228258Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
ts=2026-05-11T12:28:59.164406297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:29:01.752295913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:01.97554053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:29:01.98235488Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:29:02.378248082Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:29:05.753365518Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=warn ts=2026-05-11T12:29:06.751967475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:08.337059061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:29:08.352645925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:29:09.130481188Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-11T12:29:09.130506477Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=debug ts=2026-05-11T12:29:11.466766345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:29:11.468742583Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:29:11.730142285Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:29:11.731206949Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:29:11.751961205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:29:11.751993826Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:29:11.751995521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:11.752014601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:29:12.258689838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:48336"
level=debug ts=2026-05-11T12:29:12.504246737Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=info ts=2026-05-11T12:29:13.818773112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:29:13.828648466Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-11T12:29:15.354339156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34046"
level=debug ts=2026-05-11T12:29:15.741880675Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:29:15.878333288Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:29:16.752727599Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:19.253016258Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-05-11T12:29:21.752848775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:22.019486671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:29:22.024724946Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:29:22.629173626Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:29:22.629203472Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:29:26.003501641Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=warn ts=2026-05-11T12:29:26.752602002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:28.41178401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:29:28.417695409Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-11T12:29:29.166251885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:29:29.378802389Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-11T12:29:31.75269077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:32.752865842Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=debug ts=2026-05-11T12:29:36.128646203Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-11T12:29:36.752760313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:29:36.840926345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:29:36.84993278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:29:39.503646116Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=info ts=2026-05-11T12:29:41.752607751Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:29:41.752643581Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:29:41.752653504Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:29:41.752677325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:42.878689005Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
ts=2026-05-11T12:29:45.356234355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56972"
level=debug ts=2026-05-11T12:29:45.583581005Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:29:45.679711326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:29:45.689404428Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:29:46.253486983Z caller=checkpoint.go:372 msg="writing series" size="449 kB"
level=debug ts=2026-05-11T12:29:46.253510317Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-11T12:29:46.752282625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:49.628042115Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=info ts=2026-05-11T12:29:50.66842644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:29:50.673009142Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:29:50.673036425Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:29:50.673072002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:29:50.677180459Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:29:51.751867965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:53.003324108Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T12:29:54.385969789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:29:54.394353559Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:29:56.378393298Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-05-11T12:29:56.752890295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:29:58.434121979Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:29:59.167810282Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:29:59.753662311Z caller=checkpoint.go:372 msg="writing series" size="662 kB"
level=info ts=2026-05-11T12:30:00.036222192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:30:00.040824849Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=warn ts=2026-05-11T12:30:01.752671687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:30:02.257972941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:30:02.264131461Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:30:03.129222679Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-05-11T12:30:03.129259188Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-11T12:30:05.932612755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:30:05.93874606Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:30:06.503070524Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-05-11T12:30:06.752420919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:30:07.958173644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:30:07.966950336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:30:09.87824362Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=debug ts=2026-05-11T12:30:11.730058211Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:30:11.731120096Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:30:11.731330019Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T12:30:11.731378647Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976113 n=1
level=debug ts=2026-05-11T12:30:11.73142085Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976113 n=1
level=warn ts=2026-05-11T12:30:11.75267978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:30:11.752688693Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:30:11.75271439Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:30:11.752724911Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:30:12.26089983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:38218"
level=debug ts=2026-05-11T12:30:13.254370144Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:30:13.254394347Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
ts=2026-05-11T12:30:15.243158535Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57678"
level=debug ts=2026-05-11T12:30:16.627662565Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-11T12:30:16.719124097Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:30:16.75223123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:20.003291244Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-11T12:30:21.752434355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:30:22.93334919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:30:22.940897048Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:30:23.399040751Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:30:23.399083904Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T12:30:24.482097504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:30:24.490981394Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:30:26.752275032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:26.753390653Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:30:29.169452622Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:30:30.128777216Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=warn ts=2026-05-11T12:30:31.752881745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:33.503631488Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T12:30:36.752009431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:36.878078547Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:30:38.398441053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:30:38.407863887Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:30:40.253152094Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-05-11T12:30:41.071420202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:30:41.079182067Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:30:41.752757325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:30:41.752796027Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:30:41.752807501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:30:41.752804822Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:43.628150381Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-11T12:30:46.752335632Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:46.909930473Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:30:47.003224642Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-11T12:30:48.651728174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:30:48.659769593Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:30:48.88109365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:30:49.205084969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:30:49.20931499Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:30:50.378971009Z caller=checkpoint.go:372 msg="writing series" size="606 kB"
level=debug ts=2026-05-11T12:30:50.378997531Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-11T12:30:51.517247062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:30:51.525945751Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:30:51.751942737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:30:52.886638979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:30:52.89192976Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:30:52.891956536Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:30:52.891983505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:30:52.896310067Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:30:53.753327918Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-11T12:30:56.752463677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:30:57.128645927Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-11T12:30:59.171021173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:31:00.503221825Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:31:01.752261326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:03.878611119Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-11T12:31:06.752481205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:07.253406067Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:31:10.627938206Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T12:31:11.730799796Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:31:11.731304471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:31:11.752854105Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:31:11.752857452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:11.752886106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:31:11.752895992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:31:11.83847129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:31:11.843994604Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=info ts=2026-05-11T12:31:12.962950634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:31:12.971950344Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:31:14.003126096Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-11T12:31:15.361524122Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39448"
level=warn ts=2026-05-11T12:31:16.752842346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:17.378355164Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-11T12:31:20.753430247Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-11T12:31:21.366157356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:31:21.370805758Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:31:21.751864089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:22.603367987Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:31:24.12876876Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:31:24.128794071Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:31:24.688630956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:31:24.693403445Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:31:26.666043628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:31:26.673943749Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:31:26.673969975Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:31:26.674002512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:31:26.682712413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:31:26.752370617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:27.502916644Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:31:29.172526215Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:31:30.068905721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="59 MB" avg_comp="2.8 MB" total_uncomp="482 MB" avg_uncomp="23 MB" full=20 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:31:30.078091364Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:31:30.879170686Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:31:30.879198126Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-11T12:31:31.752861951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:34.253120557Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=warn ts=2026-05-11T12:31:36.752404456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:37.628620371Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-11T12:31:37.959638167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:31:37.968883054Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:31:39.457414224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:31:39.467001013Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:31:41.003363178Z caller=checkpoint.go:372 msg="writing series" size="749 kB"
level=info ts=2026-05-11T12:31:41.752480329Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:31:41.752511207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:31:41.752495724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:31:41.752520066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:31:42.18310264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:31:42.187900001Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
ts=2026-05-11T12:31:42.265820757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42470"
level=debug ts=2026-05-11T12:31:43.847816782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:31:44.378860836Z caller=checkpoint.go:372 msg="writing series" size="763 kB"
level=debug ts=2026-05-11T12:31:44.378888152Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-11T12:31:45.159881026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:31:45.167337713Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-11T12:31:45.363634392Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50294"
level=warn ts=2026-05-11T12:31:46.752285686Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:47.754385782Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-11T12:31:47.754413454Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=1
level=debug ts=2026-05-11T12:31:51.127679922Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:31:51.751951494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:53.161686869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:31:54.503135129Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-11T12:31:56.751901092Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:31:57.878113132Z caller=checkpoint.go:372 msg="writing series" size="403 kB"
ts=2026-05-11T12:31:59.174087148Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:32:01.253749547Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=warn ts=2026-05-11T12:32:01.752256921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:04.628408241Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T12:32:05.341907246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:32:05.352333763Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:32:06.751911518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:08.008255868Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-05-11T12:32:08.008283255Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=6
level=info ts=2026-05-11T12:32:10.983289171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:32:10.992901615Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:32:11.168475482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:32:11.173166438Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:32:11.377901805Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-05-11T12:32:11.729385587Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:32:11.730436024Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:32:11.75247652Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:32:11.752512711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:32:11.752522623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:32:11.75254904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:32:12.267783213Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54620"
level=info ts=2026-05-11T12:32:14.151624538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:32:14.155805246Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:32:14.753180903Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-11T12:32:15.365182427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56262"
level=warn ts=2026-05-11T12:32:16.751941235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:18.127721714Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-11T12:32:20.330723404Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:32:21.503744767Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-11T12:32:21.752662697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:23.417000368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:32:24.878324781Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:32:26.752253591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:32:28.192382534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:32:28.20276427Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:32:28.253177517Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
ts=2026-05-11T12:32:29.175739232Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:32:29.498232814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="62 MB" avg_comp="2.8 MB" total_uncomp="505 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:32:29.50724759Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:32:31.62821514Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T12:32:31.628241358Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T12:32:31.752786366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:32:33.233811549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:32:33.23949715Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=info ts=2026-05-11T12:32:34.823895638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:32:34.828079238Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:32:34.828105777Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:32:34.828136318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:32:34.832358539Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:32:35.018805575Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-11T12:32:35.018846625Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-05-11T12:32:36.752593677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:38.378364065Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-11T12:32:41.752771241Z caller=checkpoint.go:384 msg="flushing series" totalSize="787 kB" series=1
level=info ts=2026-05-11T12:32:41.752818103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:32:41.752835382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:32:41.752844308Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:32:41.75285248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:32:41.758373984Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-11T12:32:41.760066849Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007509422s
ts=2026-05-11T12:32:45.366822012Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55980"
level=info ts=2026-05-11T12:32:45.913699655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:32:45.925115715Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:32:46.751916427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:32:51.751858045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:32:54.385401532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:32:54.390405026Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:32:54.767821496Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:32:56.195241399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:32:56.199919426Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:32:56.554083235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.6 MB" avg_comp="2.5 MB" total_uncomp="67 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:32:56.563321804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:32:56.752274512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:32:59.177677447Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:33:01.752961815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:33:06.752561528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:11.626417908Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:33:11.628164401Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:33:11.729560048Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:33:11.730587867Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:33:11.751848156Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:33:11.751884249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:33:11.751889226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:11.751910459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:33:11.751921696Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:33:11.752032919Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-11T12:33:11.752042054Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:33:11.752061956Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.064µs success=true
level=debug ts=2026-05-11T12:33:11.752292145Z caller=checkpoint.go:372 msg="writing series" size="9.3 kB"
level=info ts=2026-05-11T12:33:14.083399702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:33:14.091526567Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:33:15.006628021Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-05-11T12:33:15.368173056Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37606"
level=warn ts=2026-05-11T12:33:16.751905354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:18.259585218Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:33:20.338412678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:33:20.346516523Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:33:21.511428212Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:33:21.752750249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:33:23.799044745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:33:23.804300999Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:33:24.765456609Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:33:26.100848721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:33:26.187234742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:33:26.197030511Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:33:26.752204697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:28.018460497Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=debug ts=2026-05-11T12:33:28.018483834Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-11T12:33:29.178936746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:33:31.270728562Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-11T12:33:31.752533495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:33:34.296503219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:33:34.301632336Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:33:34.524463861Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-11T12:33:36.751954248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:37.776617761Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-11T12:33:39.323883153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:33:39.333458721Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:33:40.406979704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:33:40.415341179Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:33:41.030482015Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:33:41.752506373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:33:41.752546626Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:33:41.752557868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:33:41.752546091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:33:42.271100451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34426"
level=debug ts=2026-05-11T12:33:44.354951861Z caller=checkpoint.go:372 msg="writing series" size="66 MB"
level=debug ts=2026-05-11T12:33:44.354987052Z caller=checkpoint.go:384 msg="flushing series" totalSize="66 MB" series=5
level=info ts=2026-05-11T12:33:45.967229102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:33:45.972094872Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:33:46.752225145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:47.535727669Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=debug ts=2026-05-11T12:33:50.79014107Z caller=checkpoint.go:372 msg="writing series" size="837 kB"
level=debug ts=2026-05-11T12:33:51.222019396Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:33:51.752343821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:54.051338499Z caller=checkpoint.go:372 msg="writing series" size="6.5 MB"
level=debug ts=2026-05-11T12:33:54.051380151Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.3 MB" series=3
level=info ts=2026-05-11T12:33:55.143115202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:33:55.14844296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:33:56.75189593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:33:57.295727968Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
ts=2026-05-11T12:33:59.180595166Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:34:00.548557112Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T12:34:01.752433735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:03.800930974Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:34:04.802251632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:34:04.807677876Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:34:04.833288909Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:34:06.752853731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:07.054670772Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-11T12:34:09.565889505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:34:09.576676804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:34:10.31145238Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-11T12:34:10.311483008Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-05-11T12:34:11.668355009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:34:11.676671325Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:34:11.676702924Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:34:11.676735203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:34:11.685725683Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:34:11.729779562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:34:11.730821727Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:34:11.752615424Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:34:11.752658452Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:34:11.75266716Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:34:11.752685995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:13.56065606Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T12:34:15.371164278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:48770"
level=warn ts=2026-05-11T12:34:16.751881685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:16.81329772Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-05-11T12:34:20.066467518Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-11T12:34:21.75286466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:23.320062899Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:34:23.320089684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:34:24.160597202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:34:26.038107378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:34:26.061242677Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:34:26.065905952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:34:26.071557537Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:34:26.572607418Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=warn ts=2026-05-11T12:34:26.752171056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:34:29.182147912Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:34:29.825425633Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=warn ts=2026-05-11T12:34:31.752653468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:33.077776032Z caller=checkpoint.go:372 msg="writing series" size="3.4 kB"
level=info ts=2026-05-11T12:34:33.315463657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:34:33.325640974Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:34:33.697487778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:34:33.702943379Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:34:36.331405147Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-11T12:34:36.752775151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:34:38.732961487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:34:38.74394035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:34:38.855967986Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:34:39.584673092Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=info ts=2026-05-11T12:34:40.35259678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:34:40.357519928Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:34:41.751991394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:34:41.752033979Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:34:41.752046571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:34:41.752073047Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:34:42.274346464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36000"
level=debug ts=2026-05-11T12:34:42.837693503Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
ts=2026-05-11T12:34:45.256381104Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38852"
level=debug ts=2026-05-11T12:34:46.090609194Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-05-11T12:34:46.090645087Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:34:46.752932013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:49.343828398Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=info ts=2026-05-11T12:34:51.257942616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:34:51.273384001Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:34:51.273417615Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:34:51.273472703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:34:51.282796092Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:34:51.752535205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:34:52.596258764Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-11T12:34:55.861333476Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T12:34:56.752833292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:34:58.089867356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:34:58.096192686Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:34:58.62442765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:34:59.1020808Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-11T12:34:59.183989229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:35:01.753892015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:02.355934478Z caller=checkpoint.go:372 msg="writing series" size="487 kB"
level=debug ts=2026-05-11T12:35:02.35596192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-11T12:35:04.873344027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:35:04.883184947Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:35:05.60986828Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=warn ts=2026-05-11T12:35:06.752356746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:35:08.481829661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:35:08.491054273Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:35:08.861155111Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T12:35:11.730258048Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:35:11.730805956Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:35:11.752545087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:35:11.752615158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:35:11.752623796Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:35:11.752643584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:12.114045301Z caller=checkpoint.go:372 msg="writing series" size="235 kB"
level=debug ts=2026-05-11T12:35:12.114069509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-11T12:35:15.372841013Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:35:15.372873174Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=info ts=2026-05-11T12:35:15.796670686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:35:15.802942913Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:35:16.752604261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:18.619833614Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T12:35:20.968719594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:35:20.979004375Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:35:20.979033993Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:35:20.979065139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:35:20.987639957Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:35:21.323055316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:35:21.332851537Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:35:21.752125824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:21.873465361Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:35:21.918756168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:35:21.923175646Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:35:24.883821958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:35:24.888067882Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:35:24.888094712Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:35:24.888122964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:35:24.891705544Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:35:25.14438153Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:35:25.144411486Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=warn ts=2026-05-11T12:35:26.752641732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:28.379537347Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-11T12:35:28.87859689Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:35:29.185367355Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:35:31.632370979Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-11T12:35:31.752972569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:32.603014637Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:35:34.885317742Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-05-11T12:35:36.752756116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:35:37.289175032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:35:37.300432407Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:35:38.138658272Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:35:38.348283819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:35:38.353115799Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:35:38.564725371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:35:38.574357864Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:35:41.390927705Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=info ts=2026-05-11T12:35:41.752206208Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:35:41.752221593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:41.752236969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:35:41.752245575Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:35:44.644841775Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=warn ts=2026-05-11T12:35:46.753219738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:47.897625357Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-11T12:35:51.151514468Z caller=checkpoint.go:372 msg="writing series" size="644 kB"
level=debug ts=2026-05-11T12:35:51.151539021Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-11T12:35:51.464532926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="2.9 MB" total_uncomp="40 MB" avg_uncomp="20 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:35:51.476636325Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:35:51.752514752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:35:52.092877863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:35:52.099203148Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:35:54.403061626Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-05-11T12:35:56.752994178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:35:56.759190644Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:35:57.65650126Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-11T12:35:59.186394842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:36:00.909481541Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:36:01.752485597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:04.162445548Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-11T12:36:06.752615469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:36:07.369170595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:36:07.37977604Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:36:07.394828484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:36:07.399989463Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:36:07.415122397Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:36:09.409777226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:36:09.414379363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:36:09.414411024Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:36:09.414440591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:36:09.418450056Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:36:09.871713967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:36:09.87600427Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:36:10.668268479Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-11T12:36:11.277117543Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:36:11.72965412Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:36:11.730717235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:36:11.752333901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:36:11.752363008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:36:11.752371745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:36:11.75239836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:36:12.278401293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41992"
level=debug ts=2026-05-11T12:36:13.921873354Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-11T12:36:16.752703833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:17.174374785Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:36:18.010014429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:36:18.018826213Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:36:20.42728393Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-11T12:36:21.75289325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:23.680982395Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:36:23.681007117Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:36:24.997335465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:36:25.008093686Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:36:26.75275994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:36:26.827994829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:36:26.832934632Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:36:26.933908232Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
ts=2026-05-11T12:36:29.188191096Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:36:30.188250859Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:36:30.188273838Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-11T12:36:31.538643563Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:36:31.752812609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:36:33.375038886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:36:33.383607338Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:36:33.439803137Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-11T12:36:36.693040299Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-11T12:36:36.7522829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:39.946121197Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=warn ts=2026-05-11T12:36:41.752212987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:36:41.752198294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:36:41.752239739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:36:41.752248136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:36:42.811192479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:36:42.818298822Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=debug ts=2026-05-11T12:36:43.199491328Z caller=checkpoint.go:372 msg="writing series" size="926 kB"
level=debug ts=2026-05-11T12:36:43.199529538Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=debug ts=2026-05-11T12:36:46.453939681Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-11T12:36:46.453966493Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=1
level=warn ts=2026-05-11T12:36:46.752268105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:49.704752777Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-11T12:36:50.767591621Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:36:50.76950642Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T12:36:51.730588463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:36:51.734533241Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:36:51.752843866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:36:52.957291755Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-11T12:36:56.21058999Z caller=checkpoint.go:372 msg="writing series" size="464 kB"
level=info ts=2026-05-11T12:36:56.707515254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:36:56.711748189Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:36:56.752675783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:36:59.189963291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:36:59.463794175Z caller=checkpoint.go:372 msg="writing series" size="354 kB"
level=debug ts=2026-05-11T12:37:00.958647965Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:37:01.751913594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:02.717059604Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T12:37:05.371932267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:37:05.378392614Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:37:05.976073135Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-11T12:37:05.976101676Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
level=warn ts=2026-05-11T12:37:06.75253578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:37:07.325044737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:37:07.33274005Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:37:09.222668118Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=debug ts=2026-05-11T12:37:11.729851923Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:37:11.730904934Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:37:11.751974091Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:37:11.752007976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:37:11.752015813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:37:11.75205598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:12.475715385Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:37:15.105919205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:37:15.114504645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:37:15.728495317Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-05-11T12:37:16.752929489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:37:17.093730436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:37:17.102742175Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:37:18.981726479Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-11T12:37:19.740393125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:37:19.749306924Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:37:21.752560152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:22.234562378Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:37:25.488367318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:37:25.48838208Z caller=checkpoint.go:372 msg="writing series" size="535 kB"
level=debug ts=2026-05-11T12:37:25.488398643Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-11T12:37:26.752345522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:28.740900432Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-11T12:37:29.191480458Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:37:31.751909769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:32.009472883Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-11T12:37:32.009512433Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=2
level=debug ts=2026-05-11T12:37:35.248106577Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=warn ts=2026-05-11T12:37:36.752033478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:37:37.404480721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b823ca12612b33e immediate=false num_chunks=1 total_comp="750 kB" avg_comp="750 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:37:37.408999718Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b823ca12612b33e/19e16df1b38%3A19e16eff29f%3A8f35f96f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-qblt\"}"
level=error ts=2026-05-11T12:37:37.409026156Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:37:37.409052209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:37:37.412693728Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:37:38.500116845Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T12:37:39.542543569Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:37:41.752290432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:37:41.752295672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=warn ts=2026-05-11T12:37:41.752307041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:37:41.752329559Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:37:41.752343358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:37:41.767271324Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-11T12:37:41.769521784Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.017294858s
ts=2026-05-11T12:37:45.38084397Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:43200"
level=info ts=2026-05-11T12:37:46.742518306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:37:46.747047238Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:37:46.752146615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:37:47.705399578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:37:47.713036717Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:37:47.713066711Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:37:47.71309957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:37:47.721042852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:37:47.734400457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:37:47.738316141Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:37:51.752664837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:37:52.442450121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:37:52.446660409Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:37:55.528951231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:37:55.53418424Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:37:56.26674618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:37:56.752183426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:37:59.193085607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:38:00.52463597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:38:00.532915305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:38:01.751925815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:03.956417682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:38:03.965378847Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:38:06.752150231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:08.604472477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:38:08.615459889Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:38:11.402510736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:38:11.410463052Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:38:11.73033277Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:38:11.730334781Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:38:11.752274812Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:38:11.752326379Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:38:11.752353115Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:38:11.752365372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:38:11.752374831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:11.752435274Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-11T12:38:11.752443912Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:38:11.752462498Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.564µs success=true
level=debug ts=2026-05-11T12:38:11.752683907Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-11T12:38:12.283753429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:44618"
level=debug ts=2026-05-11T12:38:15.005925733Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
ts=2026-05-11T12:38:15.26737856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:47560"
ts=2026-05-11T12:38:15.382045003Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:57134"
level=warn ts=2026-05-11T12:38:16.752790089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:18.249047446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:38:18.253671209Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:38:18.260036899Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-05-11T12:38:21.512608076Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:38:21.751955564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:38:23.496533926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:38:24.045858377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:38:24.054043895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:38:24.765607758Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-11T12:38:26.751996419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:38:28.01874788Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-11T12:38:28.028039927Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:38:29.194955407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:38:29.951975983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:38:29.9562172Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:38:31.271351295Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-11T12:38:31.271374742Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:38:31.751925582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:34.201870069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:38:34.205980822Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:38:34.524048527Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-11T12:38:36.752842489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:38:37.777691846Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-11T12:38:41.030513951Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:38:41.752360552Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:38:41.75239546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:38:41.752405547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:38:41.752412316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:38:42.285111848Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52830"
level=debug ts=2026-05-11T12:38:44.398007585Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T12:38:44.398074499Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=4
ts=2026-05-11T12:38:45.383542043Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37854"
level=warn ts=2026-05-11T12:38:46.752262997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:38:47.536290762Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=info ts=2026-05-11T12:38:48.747903826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:38:48.768275787Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:38:50.100070715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:38:50.115485222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:38:50.791010951Z caller=checkpoint.go:372 msg="writing series" size="999 kB"
level=warn ts=2026-05-11T12:38:51.751911273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:38:53.968371569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="28 kB" avg_uncomp="28 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:38:53.968443085Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-11T12:38:53.977436227Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:38:54.064174133Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-11T12:38:54.064217206Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=3
level=info ts=2026-05-11T12:38:54.269561629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:38:54.281851321Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:38:54.568971332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:38:54.580932084Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:38:54.869297273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="161 kB" avg_uncomp="161 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:38:54.877322094Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:38:56.752881074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:38:57.29586549Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
ts=2026-05-11T12:38:59.196344382Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:38:59.31865011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:38:59.334485596Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:39:00.548184581Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-11T12:39:01.751938099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:02.713078288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:39:02.730000638Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:39:03.80168304Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:39:04.906827962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:39:04.917786343Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:39:05.721386665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:39:05.741279369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:39:06.752236071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:39:07.05544568Z caller=checkpoint.go:372 msg="writing series" size="204 kB"
level=debug ts=2026-05-11T12:39:10.310153998Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T12:39:10.310183137Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-05-11T12:39:11.205292161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:39:11.20983225Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:39:11.730236901Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:39:11.730460234Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:39:11.752463685Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:39:11.752501693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:39:11.752510533Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:39:11.752561662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:12.632898764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:39:12.64268944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:39:12.919239023Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:39:13.043619382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:39:13.047276499Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:39:13.561408178Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=info ts=2026-05-11T12:39:14.017644685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:39:14.02120079Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-11T12:39:15.270619923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59132"
ts=2026-05-11T12:39:15.385512587Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56412"
level=warn ts=2026-05-11T12:39:16.752773564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:39:16.813992044Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-11T12:39:18.281243613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=debug ts=2026-05-11T12:39:18.281290815Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-11T12:39:18.285070769Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:39:18.582047283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:39:18.585591755Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:39:18.882047234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:39:18.88547264Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:39:19.182272602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:39:19.185694683Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:39:20.06663545Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-11T12:39:21.531003726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:39:21.535631693Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:39:21.752317772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:22.868776171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:39:22.873020988Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:39:22.873053881Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:39:22.873087092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:39:22.876747498Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:39:23.319990059Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:39:23.320013516Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-11T12:39:26.572327683Z caller=checkpoint.go:372 msg="writing series" size="356 kB"
level=warn ts=2026-05-11T12:39:26.752463285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:39:29.200400828Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:39:29.825245585Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=info ts=2026-05-11T12:39:30.933172702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:39:30.938696651Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:39:31.45007587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:39:31.453556205Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T12:39:31.752606124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:32.733261649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=info ts=2026-05-11T12:39:32.734306709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:39:32.737078843Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:39:32.737095142Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:39:32.986717498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:39:32.990353846Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:39:33.078641825Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T12:39:33.32875986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:39:33.332223804Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:39:34.926383066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:39:34.929893432Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:39:35.429239898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:39:35.433250074Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:39:35.591626423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:39:35.595056348Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:39:36.331429207Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-11T12:39:36.752599766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:37.088706583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:39:37.092341391Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:39:37.67783672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:39:37.686109058Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:39:39.58445875Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-11T12:39:41.75202616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:39:41.752048097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:39:41.752061369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:39:41.752072676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:39:42.837443117Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-11T12:39:43.778178023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:39:43.78253959Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:39:46.091102035Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T12:39:46.0911287Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:39:46.752827477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:39:49.085371963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:39:49.343742098Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=warn ts=2026-05-11T12:39:51.752778068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:39:52.597223371Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T12:39:52.81534654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:39:52.823648593Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:39:54.64497616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:39:54.653599903Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:39:55.849421883Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T12:39:56.752538082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:39:57.477991588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:39:57.482961733Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:39:59.102543368Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T12:39:59.202252435Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:39:59.873864413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:40:01.047528959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:40:01.055230086Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:40:01.75262579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:01.900976913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:40:01.904737478Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:40:02.355990223Z caller=checkpoint.go:372 msg="writing series" size="534 kB"
level=debug ts=2026-05-11T12:40:02.356028977Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-11T12:40:03.068654643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:40:03.072398971Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:40:03.719332858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:40:03.72293636Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:40:04.072024345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:40:04.076067342Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:40:05.266722657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:40:05.269996506Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:40:05.609784361Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=warn ts=2026-05-11T12:40:06.752473958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:40:08.861497887Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T12:40:11.730240552Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:40:11.730244851Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=warn ts=2026-05-11T12:40:11.752839823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:11.752838823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:40:11.752867289Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:40:11.752875029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:40:12.11446554Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=debug ts=2026-05-11T12:40:12.114488353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-11T12:40:15.36908583Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:40:15.369108731Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-11T12:40:16.752862907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:17.476370519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:40:17.482834965Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:40:18.620378249Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T12:40:19.351694635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:40:19.356353598Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:40:20.392858323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:40:20.396200171Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:40:21.75264885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:40:21.87385784Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:40:24.189017135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:40:24.192958762Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:40:24.657484355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:40:25.145439985Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:40:25.145476619Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=warn ts=2026-05-11T12:40:26.751946057Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:28.175533479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:40:28.184170565Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:40:28.379762339Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:40:29.204044991Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:40:31.632953257Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=warn ts=2026-05-11T12:40:31.752136386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:33.212211409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:40:33.216632015Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:40:33.265707596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:40:33.27023328Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:40:34.88612149Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-11T12:40:36.752815672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:40:38.13925708Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:40:39.119070321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:40:39.128555759Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:40:39.128593904Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:40:39.128623888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:40:39.136642421Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:40:41.391774825Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=info ts=2026-05-11T12:40:41.752121165Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:40:41.752157435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:40:41.752165127Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:40:41.752161758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:40:44.645078099Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
ts=2026-05-11T12:40:45.390882196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:37430"
level=warn ts=2026-05-11T12:40:46.752171201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:40:47.183024435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:40:47.184969974Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:40:47.897906235Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T12:40:48.004310168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:40:48.009983962Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:40:48.237701944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:40:48.241949835Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:40:49.079502061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:40:49.083624531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:40:50.478945057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:40:50.482424548Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:40:51.153114906Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=debug ts=2026-05-11T12:40:51.153139913Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-11T12:40:51.342637953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:40:51.350777183Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:40:51.678861779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:40:51.75208776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:52.425233774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:40:52.428864163Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:40:52.685325881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:40:52.694405827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:40:54.403788417Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-05-11T12:40:56.752171784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:40:56.984200507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:40:56.992164523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:40:57.656786646Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-11T12:40:59.205987037Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:41:00.910099275Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:41:01.752656715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:04.162643516Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-11T12:41:04.916617292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:41:04.92107727Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T12:41:06.752618945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:07.415668886Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:41:07.47468628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:41:07.478535998Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:41:07.488617629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:41:07.492335842Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:41:10.66887463Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T12:41:11.730318503Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:41:11.73032776Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:41:11.752243849Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:41:11.752289589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:41:11.752307936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:41:11.752327327Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:41:12.940685651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:41:12.944445613Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:41:13.474005396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:41:13.483527947Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:41:13.483580119Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:41:13.483641697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:41:13.492074821Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:41:13.496252641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.8 MB" avg_comp="2.9 MB" total_uncomp="58 MB" avg_uncomp="19 MB" full=3 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:41:13.504148785Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:41:13.922022102Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-11T12:41:16.752385398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:17.174766493Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:41:18.025100716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:41:18.031446216Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:41:18.196895992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:41:18.201014439Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:41:20.428120325Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-11T12:41:21.752151582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:23.497204667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:41:23.681336215Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:41:23.681360809Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-05-11T12:41:26.752383376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:26.934023053Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-11T12:41:29.207898404Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:41:29.669212782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:41:30.098715244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:41:30.108553252Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:41:30.187488102Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:41:30.187520633Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-11T12:41:31.325999987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:41:31.330209142Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:41:31.752031374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:33.439736096Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:41:34.922391859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:41:34.931282659Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:41:36.693042145Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-11T12:41:36.752370985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:41:39.050664633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:41:39.05470773Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:41:39.94630637Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T12:41:41.751989484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:41:41.75203137Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:41:41.75204031Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:41:41.752068786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:43.199791962Z caller=checkpoint.go:372 msg="writing series" size="956 kB"
level=debug ts=2026-05-11T12:41:43.199817933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-11T12:41:43.625376397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:41:43.629533552Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:41:43.725784923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:41:43.730438628Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:41:45.364387079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:41:45.373098486Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:41:45.393139769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
ts=2026-05-11T12:41:45.393893529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55724"
level=error ts=2026-05-11T12:41:45.397218914Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:41:45.70208727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:41:45.707375395Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:41:46.454271002Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-11T12:41:46.454295779Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
level=warn ts=2026-05-11T12:41:46.752108576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:49.704783164Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:41:51.752290862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:41:52.41726408Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:41:52.417329278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:41:52.424203398Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:41:52.958062518Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-11T12:41:56.211333329Z caller=checkpoint.go:372 msg="writing series" size="532 kB"
level=warn ts=2026-05-11T12:41:56.752778901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:41:58.078308417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:41:58.082359383Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:41:58.750014595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:41:58.754066396Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-11T12:41:59.209883383Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:41:59.464097899Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=warn ts=2026-05-11T12:42:01.752402236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:42:02.134078992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:42:02.139100864Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:42:02.139136088Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:42:02.139166307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:42:02.143312476Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:42:02.716780545Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T12:42:04.322265386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:42:04.325926595Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:42:05.223399915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:42:05.230046672Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:42:05.554019586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:42:05.562510755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:42:05.638292388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:42:05.976264168Z caller=checkpoint.go:372 msg="writing series" size="4.3 MB"
level=debug ts=2026-05-11T12:42:05.976313441Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=6
level=info ts=2026-05-11T12:42:06.277234872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:42:06.280965003Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T12:42:06.752670858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:09.223157423Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-11T12:42:10.249946933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:42:10.258595856Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:42:11.729549582Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:42:11.730593153Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:42:11.752436506Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:42:11.752462182Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:42:11.752470372Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:42:11.752465105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:42:12.296290584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35192"
level=debug ts=2026-05-11T12:42:12.476241963Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:42:12.83347837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:42:12.837719901Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:42:14.488926905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:42:14.492393261Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:42:15.729493255Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=warn ts=2026-05-11T12:42:16.752535604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:18.982054253Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-11T12:42:21.752739391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:22.234918598Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:42:24.855930762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:42:24.861751618Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:42:25.488261848Z caller=checkpoint.go:372 msg="writing series" size="412 kB"
level=debug ts=2026-05-11T12:42:26.224549767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:42:26.752630086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:42:27.642916566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:42:27.646713584Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:42:28.300699006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:42:28.304647276Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:42:28.741544523Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T12:42:28.74159386Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
ts=2026-05-11T12:42:29.210962562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:42:31.117340321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:42:31.127895767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:42:31.12793048Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:42:31.12795992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:42:31.136122044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:42:31.752526753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:32.008483946Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-11T12:42:32.008536698Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=info ts=2026-05-11T12:42:32.054775769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:42:32.059237153Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:42:34.96568971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:42:34.969605678Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:42:35.248014018Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=warn ts=2026-05-11T12:42:36.752854938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:42:38.173803744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:42:38.178020674Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:42:38.499991526Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-11T12:42:38.697646154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:42:38.705704103Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:42:41.751947029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:42:41.7519618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:41.751981939Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:42:41.751992189Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:42:41.753063037Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T12:42:41.758487067Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-11T12:42:41.760553089Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007934359s
ts=2026-05-11T12:42:42.298089715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35644"
level=info ts=2026-05-11T12:42:43.349103368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:42:43.353614452Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:42:43.353641066Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:42:43.353668683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:42:43.357420626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:42:43.919977382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:42:43.927654637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:42:44.325957887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:42:45.282066263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53566"
ts=2026-05-11T12:42:45.396216578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:42206"
level=info ts=2026-05-11T12:42:46.603491777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:42:46.612093738Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:42:46.752798533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:42:48.222876942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:42:48.227638847Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:42:50.696054235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:42:50.699792289Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:42:51.75237158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:42:52.572763067Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:42:56.235601381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:42:56.243046453Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:42:56.751927246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:42:57.38304859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:42:57.390254063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:42:57.876622948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:42:57.880119377Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:42:59.201967909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:42:59.206240855Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-11T12:42:59.212559634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:43:01.752065029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:43:02.088702486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:43:02.093635248Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:43:02.66984873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:43:02.674200851Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:43:06.252208603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:43:06.255836381Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:43:06.752336759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:43:11.729723194Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:43:11.730785259Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:43:11.751617269Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:43:11.751786099Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-11T12:43:11.751795697Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:43:11.751801559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-11T12:43:11.751813464Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.825µs success=true
level=debug ts=2026-05-11T12:43:11.751824658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:43:11.751835089Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:43:11.752040435Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-11T12:43:11.752048085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:43:12.300092688Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:49918"
level=debug ts=2026-05-11T12:43:15.00612191Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
ts=2026-05-11T12:43:15.283718327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58838"
level=info ts=2026-05-11T12:43:15.678266552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:43:15.686045801Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T12:43:16.752483237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:43:18.259075136Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:43:19.492063562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:43:19.500018646Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:43:19.740135678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:43:19.743430734Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:43:21.511879289Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:43:21.751971606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:43:22.924062133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:43:22.927602473Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:43:24.764480286Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=debug ts=2026-05-11T12:43:24.943949Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:43:25.673085743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:43:25.677729286Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:43:26.752536416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:43:27.920708515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:43:27.926981767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:43:28.017404333Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-11T12:43:28.375698484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:43:29.214258154Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:43:29.613986235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:43:29.618322755Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:43:31.271094934Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-11T12:43:31.271119211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:43:31.752311964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:43:32.124255585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:43:32.128180307Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:43:34.523768364Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-11T12:43:34.574998309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:43:34.579441079Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:43:36.646858817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:43:36.65492134Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:43:36.752611633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:43:37.777028704Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-11T12:43:38.676637615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:43:38.684612382Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:43:40.036860604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:43:40.042179159Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:43:40.68445237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:43:40.687836961Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:43:41.0292243Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:43:41.752208387Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:43:41.752241668Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:43:41.752250568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:43:41.752243815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:43:42.301361883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:47416"
level=debug ts=2026-05-11T12:43:44.394582931Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T12:43:44.394633764Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=4
level=warn ts=2026-05-11T12:43:46.752313041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:43:47.535514179Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-05-11T12:43:49.499073028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:43:50.796427499Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=info ts=2026-05-11T12:43:51.474230568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:43:51.489765162Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:43:51.751910655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:43:51.902682972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:43:51.920727857Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:43:54.069862155Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-11T12:43:54.069955984Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=warn ts=2026-05-11T12:43:56.752779139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:43:57.294885829Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T12:43:57.578257462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:43:57.600606258Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-11T12:43:59.216911089Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:44:00.547665574Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-11T12:44:00.770788661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:44:00.782149056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:44:01.751908875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:03.801077869Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:44:05.181581086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:44:05.199931779Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:44:06.75229961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:07.054156063Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=info ts=2026-05-11T12:44:09.885334522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:44:09.88901255Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:44:10.310221469Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T12:44:10.310247867Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-05-11T12:44:10.504846677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:44:10.509179263Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-11T12:44:10.509207211Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:44:10.509234287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:44:10.51317656Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:44:11.068283333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:44:11.071678666Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:44:11.729531666Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:44:11.73059409Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:44:11.752685408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:44:11.752721723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:44:11.75273256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:44:11.752730746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:44:12.303270732Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52156"
level=info ts=2026-05-11T12:44:13.33918497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:44:13.342459128Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:44:13.560664367Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T12:44:15.401784482Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:41186"
level=warn ts=2026-05-11T12:44:16.752388524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:16.812665695Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-11T12:44:20.065637064Z caller=checkpoint.go:372 msg="writing series" size="83 kB"
level=info ts=2026-05-11T12:44:20.347551945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:44:20.352549559Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:44:21.752481191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:23.319998093Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:44:23.320021356Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-11T12:44:23.986449792Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:44:23.998482829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:44:24.005597609Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:44:24.436865823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:44:24.445381065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:44:24.445410564Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:44:24.4454483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:44:24.453445853Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:44:26.126784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:44:26.131524357Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:44:26.571945406Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=warn ts=2026-05-11T12:44:26.751896521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:44:27.697245238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:44:27.701861765Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-11T12:44:29.221110267Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:44:29.221973901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:44:29.226561163Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:44:29.667132443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:44:29.669000104Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:44:29.825018768Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
level=info ts=2026-05-11T12:44:31.330700068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:44:31.339190279Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:44:31.752525506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:33.078295654Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T12:44:35.602320274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:44:35.610072895Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:44:36.331120515Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=warn ts=2026-05-11T12:44:36.752644943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:44:36.824872145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:44:36.828795047Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:44:37.073861976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:44:37.080907605Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:44:39.244191732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:44:39.248230969Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:44:39.584520074Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=info ts=2026-05-11T12:44:40.066081238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:44:40.069930271Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:44:41.752372146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:44:41.75240459Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:44:41.752413293Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:44:41.752415902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:42.837036724Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=info ts=2026-05-11T12:44:43.648183925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:44:43.651888944Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:44:46.090471715Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T12:44:46.090495063Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T12:44:46.752410807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:44:48.84379739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:44:48.848293519Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:44:49.343027641Z caller=checkpoint.go:372 msg="writing series" size="368 kB"
level=warn ts=2026-05-11T12:44:51.752333097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:44:52.596034746Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-11T12:44:52.92062724Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:44:55.848816747Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T12:44:56.751874416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:44:58.622791052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:44:58.634280024Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:44:59.101484686Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T12:44:59.222414611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:45:00.556194966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:45:00.560230857Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:45:01.752340618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:01.831691154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:45:01.835731495Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:45:02.35474432Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-11T12:45:02.354768529Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-05-11T12:45:05.488894817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:45:05.494488974Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:45:05.60870135Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=warn ts=2026-05-11T12:45:06.752816475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:07.245041194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:45:07.24851643Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:45:07.85823742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:45:07.863024163Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:45:08.063604398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:45:08.067447499Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:45:08.067478734Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:08.067518787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:45:08.070758994Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:45:08.860801786Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T12:45:09.51447003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:45:09.521414593Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:45:11.730194282Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:45:11.730264255Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:45:11.731610206Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T12:45:11.731657753Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976114 n=1
level=debug ts=2026-05-11T12:45:11.731697033Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976114 n=1
level=info ts=2026-05-11T12:45:11.752457279Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:45:11.752489408Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:45:11.752497824Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:45:11.752540028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:45:12.113983465Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=debug ts=2026-05-11T12:45:12.114007054Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-05-11T12:45:12.260442778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:45:12.267949065Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T12:45:12.306920991Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:59456"
level=info ts=2026-05-11T12:45:15.164641657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:45:15.168625264Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:45:15.289399175Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54452"
level=debug ts=2026-05-11T12:45:15.368840875Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:45:15.368864209Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-11T12:45:15.404445501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49798"
level=warn ts=2026-05-11T12:45:16.752535315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:16.91949135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:45:16.926916732Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:45:17.675001429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:45:17.682747519Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:45:18.619876498Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-11T12:45:21.248980982Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:45:21.254978083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:45:21.258545789Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:45:21.752849388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:45:21.873197049Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:45:23.628883966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:45:23.632370983Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:45:23.632395342Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:23.632420266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:45:23.635219864Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:45:25.143683922Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:45:25.143719133Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=warn ts=2026-05-11T12:45:26.752192648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:45:28.013843599Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:45:28.378941987Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-11T12:45:29.224273845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:45:31.632375425Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
level=warn ts=2026-05-11T12:45:31.752603606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:32.75481522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:45:32.759845733Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:45:32.759876896Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:32.759904994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:45:32.763012437Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:45:34.885205334Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-11T12:45:36.75266347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:45:38.138302699Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:45:38.365880225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:45:38.371358017Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:45:39.383416601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:45:39.387555525Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:45:39.387604818Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:39.387632634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:45:39.390699541Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:45:39.521099082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:45:39.524625751Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-11T12:45:39.524652372Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:39.687966168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:45:39.691688434Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:45:41.067063605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:45:41.075028895Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:45:41.391125982Z caller=checkpoint.go:372 msg="writing series" size="4.6 kB"
level=info ts=2026-05-11T12:45:41.752279482Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:45:41.752308039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:45:41.752316486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:45:41.752322265Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:42.270629986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:45:42.274259053Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:45:44.644545738Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-11T12:45:45.139231312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:45:45.142825284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:45:45.142854242Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:45.142879102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:45:45.145900102Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:45:45.305179661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:45:45.308646087Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:45:45.463208782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:45:45.466615185Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:45:46.752485788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:45:47.896938514Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T12:45:50.064557746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:45:50.068332404Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:45:51.151320564Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=debug ts=2026-05-11T12:45:51.151349933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-11T12:45:51.752105402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:52.282311491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:45:52.286887594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:45:52.621287586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:45:52.625426627Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:45:53.163929968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:45:53.167295128Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:45:53.167321874Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:53.167345455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:45:53.17022769Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:45:53.592469204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:45:53.599087808Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:45:54.402936469Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-11T12:45:55.788270559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:45:55.797280177Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:45:56.752109905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:45:56.926021834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:45:56.929653555Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:45:57.656210048Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=debug ts=2026-05-11T12:45:58.036622462Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T12:45:59.225990776Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:45:59.74253903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:45:59.746396765Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:45:59.746421277Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:45:59.746451709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:45:59.749649586Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:46:00.909620387Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-11T12:46:01.444224422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:46:01.448144711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:46:01.752625527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:04.162588542Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=info ts=2026-05-11T12:46:04.257783738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:46:04.263231761Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:46:05.36660477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:46:05.37059017Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:46:05.370615002Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:46:05.370642548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:46:05.37397548Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:46:06.752285118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:46:07.330203887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:46:07.333870668Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:46:07.415093868Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:46:10.081078542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:46:10.085776744Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:46:10.668557622Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T12:46:11.730221407Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:46:11.730238249Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:46:11.752120127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:46:11.752150295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:46:11.752158447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:46:11.752275113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:46:12.309237656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34376"
level=info ts=2026-05-11T12:46:13.120276279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:46:13.127928037Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:46:13.921459278Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T12:46:14.729043029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:46:14.737648991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T12:46:15.406896242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:39682"
level=warn ts=2026-05-11T12:46:16.752646876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:17.174669253Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-11T12:46:17.550626195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:46:17.554440652Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:46:18.017898037Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:46:18.192293414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:46:18.196593307Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:46:19.59971732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:46:19.603130814Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:46:20.427735245Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-11T12:46:21.75274614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:46:23.513704275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:46:23.517402934Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:46:23.680678076Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:46:23.680703252Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=debug ts=2026-05-11T12:46:24.430175686Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:46:26.082707959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:46:26.091125275Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:46:26.75214747Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:26.933705254Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-11T12:46:29.227793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:46:30.187827886Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:46:30.187849403Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-11T12:46:31.354268931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:46:31.358444405Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:46:31.752247713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:46:31.929326663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:46:31.933431307Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:46:33.069917805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:46:33.073873185Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:46:33.408848505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:46:33.414169361Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:46:33.439166869Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:46:35.166481118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:46:35.171650069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:46:36.034456429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:46:36.0378983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:46:36.692141097Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-11T12:46:36.75232284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:39.945393154Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T12:46:41.752131083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:46:41.752148476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:41.752165076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:46:41.752175631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:46:42.270845791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:46:42.274280341Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:46:43.199103348Z caller=checkpoint.go:372 msg="writing series" size="960 kB"
level=debug ts=2026-05-11T12:46:43.199131211Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-11T12:46:44.337861216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:46:44.345296438Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:46:44.621046126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:46:44.627381267Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T12:46:45.293488942Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53278"
level=debug ts=2026-05-11T12:46:46.45329273Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-11T12:46:46.453316145Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=1
level=warn ts=2026-05-11T12:46:46.752786522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:46:47.853014313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:46:47.857335678Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:46:49.70420148Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-11T12:46:51.752893844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:52.957396547Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-11T12:46:54.88733145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:46:54.896176322Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:46:54.896202948Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:46:54.896233078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:46:54.904263989Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:46:54.913308122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:46:54.916715338Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:46:56.210630617Z caller=checkpoint.go:372 msg="writing series" size="608 kB"
level=warn ts=2026-05-11T12:46:56.752746992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:46:58.733616534Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T12:46:59.229380744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:46:59.463790254Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=warn ts=2026-05-11T12:47:01.75277284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:47:02.716370769Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-11T12:47:05.97404002Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-05-11T12:47:05.974065644Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.2 MB" series=6
level=warn ts=2026-05-11T12:47:06.752609671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:07.883315803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:47:07.892087814Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:47:08.304872377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:47:08.312828248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:47:09.222595087Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-11T12:47:10.505976149Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:47:11.425066664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:47:11.435580721Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:47:11.642404527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:47:11.650689535Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:47:11.729329136Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:47:11.730395785Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:47:11.751912283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:47:11.751944424Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:47:11.751953731Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:47:11.751987785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:47:12.475130323Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:47:14.196937391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:47:14.20108606Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-11T12:47:15.295040599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:38790"
level=info ts=2026-05-11T12:47:15.434175618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:47:15.438080398Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:47:15.728131491Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=warn ts=2026-05-11T12:47:16.752519423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:17.253703942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:47:17.257755624Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:47:17.960635169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:47:17.964588016Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:47:18.981764148Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-11T12:47:20.626045043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:47:20.630393024Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:47:21.752873259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:47:22.234103855Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:47:25.488273904Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=debug ts=2026-05-11T12:47:25.488298353Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-11T12:47:26.016802533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:47:26.021546697Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:47:26.752489839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:28.669014202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:47:28.672768499Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:47:28.740410927Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-11T12:47:29.231095405Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:47:31.75186023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:47:32.008146962Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-11T12:47:32.008179622Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-11T12:47:33.367629911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:47:33.372557108Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:47:33.398691597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:47:33.408181929Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:47:34.248045298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:47:34.255269979Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:47:34.271322691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:47:34.275675176Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:47:34.578712104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:47:35.144533579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:47:35.148098355Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:47:35.247165127Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=warn ts=2026-05-11T12:47:36.752353172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:36.905693279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:47:36.912331563Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:47:38.500048848Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T12:47:41.752856038Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T12:47:41.752866493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:47:41.752891771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:47:41.752899598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:47:41.752917422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:41.758550742Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-11T12:47:41.760267638Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008320326s
level=info ts=2026-05-11T12:47:42.196212434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:47:42.200948303Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:47:44.957227721Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:47:46.752084431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:47:51.621935796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:47:51.630226611Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:47:51.630251562Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:47:51.630296981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:47:51.637993474Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:47:51.752714074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-11T12:47:56.752304961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:47:59.232466471Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=warn ts=2026-05-11T12:48:01.752750519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:02.442657885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:48:04.78738063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:48:04.797963885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:48:04.978615469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:48:04.982185268Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:48:06.752369102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:06.854717977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:48:06.862445065Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:48:07.65508681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:48:07.659591982Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:48:09.801152643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:48:09.808118136Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:48:10.920903106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:48:10.924777899Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:48:11.416054356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:48:11.419469831Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:48:11.73038285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:48:11.730788913Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:48:11.752191624Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:48:11.752217364Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:48:11.752238751Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:48:11.752246492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:48:11.752254074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:11.752483151Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-11T12:48:11.752494614Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:48:11.752515511Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.131µs success=true
level=debug ts=2026-05-11T12:48:11.752797127Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:48:14.168943049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:48:14.173040504Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:48:14.173064845Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:48:14.17309143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:48:14.176692378Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:48:14.636487272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:48:14.639788795Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:48:15.00614334Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
ts=2026-05-11T12:48:15.413167637Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36736"
level=info ts=2026-05-11T12:48:15.611183736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:48:15.618871911Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:48:16.752129869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:17.082491253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:48:17.089075115Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:48:17.089102112Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:48:17.08913272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:48:17.095679759Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:48:18.259983439Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:48:20.037691891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:48:20.041234382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:48:21.512753277Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:48:21.752745981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:22.884503386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:48:22.888105617Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:48:23.45079692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:48:23.458783215Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:48:24.765905026Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-11T12:48:25.481816319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:48:25.485848711Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:48:26.752255533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:28.018398393Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-11T12:48:28.369180584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:48:28.373328386Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:48:29.234193715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:48:29.906807796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:48:29.916033005Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:48:30.929500836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:48:30.933106467Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:48:31.271128654Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-11T12:48:31.27116422Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-11T12:48:31.605908205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:48:31.610049419Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:48:31.752633449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:31.896990286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:48:31.903452283Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:48:33.159029074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:48:33.160669187Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T12:48:34.524334011Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-11T12:48:34.647827549Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:48:36.752872787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:37.777213902Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-05-11T12:48:41.030507209Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:48:41.752579222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:48:41.752608236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:48:41.752617866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:48:41.752646382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:44.370202273Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T12:48:44.370246047Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=4
level=warn ts=2026-05-11T12:48:46.752315349Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:47.536302215Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-05-11T12:48:50.790733841Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-11T12:48:50.790784542Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=warn ts=2026-05-11T12:48:51.751899531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:48:52.627542461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:48:52.644047Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:48:52.701097057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:48:52.71809399Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:48:54.065233493Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T12:48:54.065273047Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-11T12:48:55.014644332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:48:55.035187298Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:48:55.493171517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:48:55.505407948Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:48:56.753223074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:48:57.29614845Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
ts=2026-05-11T12:48:59.237355971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:48:59.491434273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:48:59.49972402Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:49:00.086431057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:49:00.10681072Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:49:00.548391094Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-11T12:49:00.597298641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:49:00.611591366Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:49:01.368556317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:49:01.385665436Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:49:01.385706951Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:49:01.385755212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:49:01.405930102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:49:01.751906846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:03.305484513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:49:03.317174801Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:49:03.801825607Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T12:49:06.752290741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:06.92515425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:49:06.939736846Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:49:07.055470694Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=debug ts=2026-05-11T12:49:08.760538289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:49:10.310667135Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T12:49:10.31069245Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=5
level=debug ts=2026-05-11T12:49:11.729548786Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:49:11.7306008Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:49:11.752604261Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:49:11.752638961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:49:11.752647343Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:49:11.752687173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:12.837798389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:49:12.841889868Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:49:13.561533195Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=info ts=2026-05-11T12:49:13.919878138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:49:13.92922109Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:49:13.953277255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:49:13.96202228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:49:14.497499996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:49:14.501455442Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:49:14.934153472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:49:14.942336557Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-11T12:49:15.300447146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40666"
level=warn ts=2026-05-11T12:49:16.752504869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:16.813671543Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-11T12:49:20.066435445Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-05-11T12:49:21.412613841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:49:21.41689349Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:49:21.752870798Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:21.976854306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:49:21.981036644Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:49:23.218608138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:49:23.229816821Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:49:23.321019689Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:49:23.321051728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=debug ts=2026-05-11T12:49:26.572935013Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=warn ts=2026-05-11T12:49:26.751891871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:28.187460382Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T12:49:29.240796917Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:49:29.826089451Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-05-11T12:49:31.752836288Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:33.078911761Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T12:49:34.511101832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:49:34.516623283Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:49:35.643600526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:49:36.331757009Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-11T12:49:36.752805981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:39.584595709Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=info ts=2026-05-11T12:49:40.005620425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:49:40.01060157Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:49:40.36663205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:49:40.37753029Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:49:41.752330426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:49:41.752360014Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:49:41.752368362Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:49:41.752362534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:42.153267646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:49:42.158257058Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:49:42.838067223Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=debug ts=2026-05-11T12:49:42.838098207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-11T12:49:43.852369207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:49:43.856020163Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:49:44.820022959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:49:44.826742734Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T12:49:45.417119162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:56266"
level=debug ts=2026-05-11T12:49:46.091351216Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-11T12:49:46.752368093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:49.344247612Z caller=checkpoint.go:372 msg="writing series" size="505 kB"
level=info ts=2026-05-11T12:49:50.056110179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:49:50.061620679Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:49:50.526152812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:49:50.530646567Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:49:50.530676036Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:49:50.530705352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:49:50.535209958Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:49:51.752775481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:49:52.596959164Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T12:49:53.566857679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:49:53.571399522Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:49:55.849614951Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T12:49:56.75230843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:49:57.779237456Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:49:57.782920632Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:49:59.102798331Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T12:49:59.24277336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:49:59.387891051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:49:59.391406266Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:50:01.75230074Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:02.356418279Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=debug ts=2026-05-11T12:50:02.356443701Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-11T12:50:03.62506762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:50:03.629952378Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:50:04.15360732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:50:04.157869226Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:50:04.566164265Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:50:04.566244156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:50:04.570209519Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T12:50:04.638437412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:50:04.646579837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:50:05.292477385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:50:05.301148124Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:50:05.589035582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:50:05.592490767Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:50:05.609228044Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=info ts=2026-05-11T12:50:06.694195784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:50:06.698745515Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:50:06.751877383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:08.862346183Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-11T12:50:11.729289018Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:50:11.730336974Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:50:11.752267318Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:50:11.752310892Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:50:11.752326568Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:50:11.752327673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:12.114479905Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=debug ts=2026-05-11T12:50:12.114506516Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-11T12:50:15.371457179Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:50:15.371493235Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-11T12:50:15.418342946Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:34106"
level=warn ts=2026-05-11T12:50:16.752806677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:18.621183838Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T12:50:20.349681263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:50:20.355645294Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T12:50:21.752737052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:21.874174937Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:50:21.908245273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:50:21.91203831Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:50:22.08533587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:50:22.093024601Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:50:22.755325249Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T12:50:25.144483915Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:50:25.144516677Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T12:50:26.324077641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:50:26.337406098Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T12:50:26.752100474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:50:28.038476699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:50:28.041921966Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:50:28.379806088Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-11T12:50:28.745181952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:50:28.751978966Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T12:50:29.244693784Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:50:31.369795781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:50:31.374316813Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:50:31.633369301Z caller=checkpoint.go:372 msg="writing series" size="231 kB"
level=warn ts=2026-05-11T12:50:31.752061726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:32.692708984Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:50:34.885655316Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-11T12:50:36.608711816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:50:36.612881084Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:50:36.752046878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:50:38.139267371Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:50:38.461096554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:50:38.46463696Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:50:38.697165151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:50:38.701143526Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:50:41.391777535Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-05-11T12:50:41.752690777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:50:41.752704572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:50:41.752730833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:50:41.752742221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:50:44.646923942Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-11T12:50:44.78264149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:50:44.79170147Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:50:44.79172906Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:50:44.791760404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:50:44.799075635Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:50:46.752741334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:50:47.772788501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:50:47.776048075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:50:47.898367226Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T12:50:50.881028759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:50:50.884842324Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:50:51.15177207Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-11T12:50:51.151797768Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-11T12:50:51.544687777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:50:51.548986911Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T12:50:51.752056198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:50:52.354500499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:50:52.358074821Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T12:50:54.354448632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:50:54.358294306Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-11T12:50:54.358332375Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:50:54.358357883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:50:54.361680728Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:50:54.403930933Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-05-11T12:50:56.752174285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:50:56.815308516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:50:56.823474611Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:50:57.656901216Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-05-11T12:50:59.246436208Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:50:59.564307478Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:51:00.910321432Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T12:51:01.7523685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:01.991588453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:51:02.000289353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:51:02.509141018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:51:02.512868801Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:51:02.623333145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:51:02.627164993Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:51:04.163272921Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-11T12:51:06.752542258Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:51:07.415883685Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:51:10.668693205Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T12:51:11.730231718Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:51:11.730554549Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:51:11.752617901Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:51:11.752647853Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:51:11.752657543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:51:11.752673417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:11.760838363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:51:11.765024437Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T12:51:12.322935893Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:51010"
level=info ts=2026-05-11T12:51:12.700909451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:51:12.709696443Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:51:13.921872505Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T12:51:14.874746119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:51:14.879063291Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-11T12:51:15.305781888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35998"
level=info ts=2026-05-11T12:51:15.342538737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:51:15.350716777Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:51:16.752459231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:16.982650462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:51:16.987020788Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:51:17.174863692Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-11T12:51:19.283900275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:51:19.289234677Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:51:19.289270545Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:51:19.289310613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:51:19.292615513Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:51:20.42856081Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-11T12:51:21.75284963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:22.569619917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:51:22.574089372Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:51:23.6819439Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:51:23.681967533Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:51:24.113405935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:51:24.12025068Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:51:25.190635161Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:51:26.160204857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:51:26.163719717Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:51:26.75245933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:51:26.93363176Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
ts=2026-05-11T12:51:29.247513841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:51:30.189029977Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:51:30.189051807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-11T12:51:31.75285939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:32.574814813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:51:32.580194374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:51:33.439722347Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T12:51:34.176130018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:51:34.184242944Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:51:34.19946048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:51:34.199527598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="12 kB" avg_uncomp="12 kB" idle=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:51:34.203009263Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:51:36.312919337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:51:36.316905434Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:51:36.693026754Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-11T12:51:36.752056358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:36.960538126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:51:36.964123063Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:51:39.325726142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:51:39.329756784Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:51:39.946893171Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T12:51:41.752513461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:51:41.752550658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:51:41.752533863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:41.752560856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:51:43.199700167Z caller=checkpoint.go:372 msg="writing series" size="982 kB"
level=debug ts=2026-05-11T12:51:43.199723222Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-11T12:51:44.72726141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:51:44.730877431Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-11T12:51:45.307507855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46122"
level=info ts=2026-05-11T12:51:45.847944397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:51:45.852207444Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:51:46.454146351Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-11T12:51:46.454170128Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=1
level=info ts=2026-05-11T12:51:46.543209872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:51:46.546668518Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:51:46.54669422Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:51:46.546723859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:51:46.549683926Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:51:46.752205164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:51:48.108681739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:51:48.114345803Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:51:49.705496989Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-11T12:51:50.840928508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:51:50.84509606Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=warn ts=2026-05-11T12:51:51.752176229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:51:52.958254706Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-11T12:51:53.021490465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:51:53.025284843Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:51:53.121697311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:51:53.128944611Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:51:55.011895285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:51:55.020101294Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:51:56.211922813Z caller=checkpoint.go:372 msg="writing series" size="482 kB"
level=warn ts=2026-05-11T12:51:56.752464277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:51:59.249013419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:51:59.464312722Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=warn ts=2026-05-11T12:52:01.751876387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:02.717452334Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-11T12:52:05.31148461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:52:05.403802979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:52:05.410211099Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:52:05.766635259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:52:05.770042359Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:52:05.976865756Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-11T12:52:05.977032116Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=6
level=warn ts=2026-05-11T12:52:06.752598478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:08.119465528Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:52:08.268852169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:52:08.272777478Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-11T12:52:08.272800787Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:08.272826561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:52:08.275855496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:52:08.703155666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:52:08.707747651Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:52:08.870559268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:52:08.87885842Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:52:09.22332215Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-11T12:52:11.729969592Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:52:11.731113516Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:52:11.751915631Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:52:11.751930789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:11.751948025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:52:11.75196572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-11T12:52:12.325937262Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:41424"
level=info ts=2026-05-11T12:52:12.443722696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:52:12.447495714Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:52:12.447527921Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:12.447554879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:52:12.450545218Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:52:12.476651298Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:52:14.489835834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:52:14.493502657Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:52:15.537145488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:52:15.540558335Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:52:15.540603983Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:15.540633211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:52:15.543405304Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:52:15.729082369Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-11T12:52:16.752241939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:18.982274151Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-11T12:52:19.20784064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:52:19.2114195Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:52:21.752669679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:52:22.192617484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:52:22.201483314Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:52:22.236050511Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:52:22.443523987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:52:22.446938438Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:52:22.446960278Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:22.446983685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:52:22.449715475Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:52:23.650942907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:52:23.65453771Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:52:25.488499616Z caller=checkpoint.go:372 msg="writing series" size="457 kB"
level=debug ts=2026-05-11T12:52:25.488522944Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-11T12:52:26.02613786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:52:26.029790411Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:52:26.02981441Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:26.029839865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:52:26.032704501Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T12:52:26.752127015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:28.741600274Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-05-11T12:52:28.852790646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:52:28.857022175Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T12:52:29.250418585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:52:29.852850127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:52:29.856181887Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:52:30.322165081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:52:30.325912668Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:52:31.082412248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:52:31.085703889Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:52:31.752818218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:32.009291435Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-11T12:52:32.009326325Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=2
level=info ts=2026-05-11T12:52:34.100128975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:52:34.110002513Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:52:34.390884876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:52:34.394859202Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:52:34.394884116Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:34.394912065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:52:34.398384964Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:52:35.247961776Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=debug ts=2026-05-11T12:52:36.14078801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:52:36.752596764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:52:38.070459798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:52:38.074230683Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:52:38.500532975Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-11T12:52:39.754802864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:52:39.759265196Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:52:39.759290808Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:52:39.75931554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:52:39.762089542Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:52:40.137222464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:52:40.140801944Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:52:40.983277637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:52:40.986812933Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T12:52:41.751948155Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:52:41.751982688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:52:41.751990893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:52:41.751993739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:41.75303344Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T12:52:41.76139206Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-11T12:52:41.763013757Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010278602s
level=info ts=2026-05-11T12:52:41.95663758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:52:41.964689306Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-11T12:52:42.327532069Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:52462"
level=info ts=2026-05-11T12:52:43.240716653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:52:43.249666777Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-11T12:52:45.424794926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51766"
level=info ts=2026-05-11T12:52:45.472486369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:52:45.475955253Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:52:46.752295174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:52:46.960056025Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:52:46.961870673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T12:52:48.299841437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:52:48.303877747Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:52:51.752034841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:52:55.569802712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:52:55.574059276Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:52:56.474364071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:52:56.4785406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:52:56.752206856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:52:59.251545386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:53:01.75223687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:53:01.836483128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:53:01.840758689Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:53:02.137304293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:53:02.142258651Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:53:02.849350662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:53:02.856908248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:53:05.669299201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:53:05.675178001Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:53:06.243496848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:53:06.247155445Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:53:06.752456448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:11.230840765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:53:11.729656513Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:53:11.730710453Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:53:11.752452467Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:53:11.752468156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:11.752494216Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:53:11.752504136Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:53:11.752495574Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T12:53:11.752712069Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-11T12:53:11.752722196Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:53:11.752740411Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.566µs success=true
level=debug ts=2026-05-11T12:53:11.752967452Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:53:12.222681826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:53:12.227190871Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:53:13.187281134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:53:13.191421731Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:53:15.0062217Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-05-11T12:53:15.045244242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:53:15.049077628Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:53:15.363167469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:53:15.366673037Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=warn ts=2026-05-11T12:53:16.75241965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:53:17.284454893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:53:17.28819708Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:53:18.259880417Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:53:19.453877619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:53:19.46021091Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:53:19.511242917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:53:19.515352824Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:53:21.512378417Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:53:21.752158901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:53:22.687414527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:53:22.691147482Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:53:24.765207391Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-11T12:53:25.034859454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="92 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:53:25.041712232Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:53:26.53718039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:53:26.542021647Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:53:26.752852369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:28.01891235Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-11T12:53:28.55853375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:53:28.565904576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-11T12:53:29.253393178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:53:31.271389391Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-11T12:53:31.271411372Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-05-11T12:53:31.752290672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:34.52451674Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T12:53:36.752852896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:37.777418657Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-11T12:53:39.435379933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:53:39.447453362Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:53:39.44750015Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:53:39.447543315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:53:39.456151108Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:53:41.030297445Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:53:41.752055668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:53:41.752068412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:41.752085435Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:53:41.752093684Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:53:41.989799045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:53:42.451702762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:53:42.456226359Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:53:44.392881021Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T12:53:44.392931788Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=4
level=info ts=2026-05-11T12:53:44.478290795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:53:44.489789209Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:53:45.814643455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:53:45.827393825Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T12:53:46.75218543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:53:47.536480622Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=debug ts=2026-05-11T12:53:48.361743681Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:53:49.393001411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:53:49.411102716Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:53:49.499197926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:53:49.520394886Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:53:50.790912316Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-11T12:53:50.790941604Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=2
level=info ts=2026-05-11T12:53:51.751771648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:53:51.751930781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-11T12:53:51.759436764Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:53:53.065016452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:53:53.076388356Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:53:54.083230964Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T12:53:54.08331571Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-11T12:53:54.423152579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:53:54.434334983Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T12:53:56.752293879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:53:56.763948329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:53:56.778387202Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T12:53:57.296539286Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T12:53:59.032156129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:53:59.04469594Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-11T12:53:59.256383139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:53:59.849662799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:53:59.868132619Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:54:00.549113776Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T12:54:01.752138113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:03.384688574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:54:03.397035944Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:54:03.799213361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:54:03.803123368Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=error ts=2026-05-11T12:54:03.810322078Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:54:05.252951814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:54:05.265157374Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T12:54:06.752242486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:54:07.055162735Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-05-11T12:54:10.310660954Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T12:54:10.310692933Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=info ts=2026-05-11T12:54:11.730228743Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:54:11.730600504Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:54:11.752275593Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:54:11.752312524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:54:11.752322373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:54:11.752343007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:54:12.332802785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:42060"
level=info ts=2026-05-11T12:54:12.556136637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:54:12.563675939Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:54:12.870511042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:54:12.874025143Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:54:13.561555077Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=debug ts=2026-05-11T12:54:14.399746325Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:54:15.121066176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:54:15.129277978Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:54:15.26250142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:54:15.270932655Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-11T12:54:15.429955352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:55008"
level=warn ts=2026-05-11T12:54:16.752772031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:16.795924977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:54:16.803663145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:54:16.813762419Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-05-11T12:54:20.066910067Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-11T12:54:21.752280287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:23.220873903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:54:23.225882443Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:54:23.320760376Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:54:23.320784623Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=debug ts=2026-05-11T12:54:26.574428955Z caller=checkpoint.go:372 msg="writing series" size="353 kB"
level=warn ts=2026-05-11T12:54:26.75217491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:28.066506072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:54:28.072699617Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:54:28.072733431Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:54:28.072765498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:54:28.076696578Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-11T12:54:29.262823009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:54:29.825610759Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-11T12:54:31.752690189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:32.251628521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:54:32.255411869Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:54:33.078485048Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-11T12:54:36.331681278Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-11T12:54:36.752422304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:54:37.782428386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:54:37.797458011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:54:37.806542998Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:54:37.906729374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:54:37.914940652Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:54:37.914972175Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:54:37.91503005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:54:37.922705864Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:54:38.668808399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:54:38.672720868Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:54:39.584716749Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-11T12:54:40.130606999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:54:40.134735584Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:54:41.752169742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:54:41.752201492Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:54:41.75221175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:54:41.752210962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:42.402904004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:54:42.406697397Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:54:42.837838207Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=info ts=2026-05-11T12:54:43.528833653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:54:43.534679132Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:54:44.407157125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:54:44.410582928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:54:46.090749222Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T12:54:46.09077389Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-05-11T12:54:46.752256531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:54:49.344050579Z caller=checkpoint.go:372 msg="writing series" size="632 kB"
level=debug ts=2026-05-11T12:54:49.535610772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:54:49.74636721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:54:49.753834683Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T12:54:51.751903466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:54:52.596823322Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T12:54:53.221517004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:54:53.226327429Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T12:54:54.046240846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:54:54.049958579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:54:54.163439077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:54:54.167297908Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T12:54:55.850247802Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T12:54:55.868400002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:54:55.875865533Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:54:56.55019426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:54:56.553546652Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:54:56.752289789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:54:56.934474628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:54:56.942326345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:54:57.241808734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:54:57.245443341Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:54:59.102617449Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T12:54:59.264720619Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=warn ts=2026-05-11T12:55:01.752695741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:02.356425519Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=debug ts=2026-05-11T12:55:02.356450751Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-05-11T12:55:04.223349499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:55:04.227828823Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:55:05.609763005Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=warn ts=2026-05-11T12:55:06.752305909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:55:06.846702754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:55:06.853424612Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:55:06.853450367Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:55:06.853477931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:55:06.860428767Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:55:08.862151345Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T12:55:10.892730912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:55:10.901611885Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:55:10.901638145Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:55:10.901668134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:55:10.902591972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:55:10.910184092Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:55:10.910183477Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:55:11.730368289Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:55:11.730380262Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:55:11.752377455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T12:55:11.752375954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:11.75240469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:55:11.752412991Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:55:12.114959779Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=debug ts=2026-05-11T12:55:12.114981943Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=debug ts=2026-05-11T12:55:15.369661709Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T12:55:15.369705444Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-11T12:55:16.752286141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:55:17.005364283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:55:17.010069006Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T12:55:17.817263549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:55:17.822170069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:55:18.610601251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:55:18.614431015Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T12:55:18.620577949Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-11T12:55:20.125995639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:55:21.7520462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:21.873495126Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T12:55:22.943470153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:55:22.950769874Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:55:23.805947201Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:55:24.578172322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:55:24.582478559Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:55:25.145999705Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T12:55:25.146035384Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T12:55:25.443017677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:55:25.446420773Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T12:55:26.752064902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:55:27.909042023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:55:27.917959107Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:55:28.307037192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:55:28.310606362Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T12:55:28.379972777Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-11T12:55:28.88418061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:55:28.888168721Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T12:55:29.266803802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T12:55:31.63353684Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-05-11T12:55:31.752815885Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:34.885923792Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-11T12:55:36.752843839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:55:37.309045595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:55:37.314023266Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T12:55:38.139125135Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-11T12:55:38.517024148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:55:38.520394004Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:55:38.719987605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:55:38.723333258Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:55:41.392309964Z caller=checkpoint.go:372 msg="writing series" size="5.1 kB"
level=info ts=2026-05-11T12:55:41.75228003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:55:41.752308739Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:55:41.752320063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:55:41.752321675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T12:55:42.336307658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:57504"
level=info ts=2026-05-11T12:55:44.423998736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:55:44.428779327Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:55:44.645781498Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
ts=2026-05-11T12:55:45.318422544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50594"
level=info ts=2026-05-11T12:55:46.427179721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:55:46.431347204Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T12:55:46.752274471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:47.898288948Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T12:55:49.282430755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:55:49.292408207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:55:50.111929051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:55:50.119083881Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:55:51.151378728Z caller=checkpoint.go:372 msg="writing series" size="698 kB"
level=debug ts=2026-05-11T12:55:51.151401526Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-11T12:55:51.751883709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:54.404444009Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-11T12:55:55.033483743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:55:55.037118733Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:55:56.034224732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:55:56.044000189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:55:56.147595154Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:55:56.752476819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:55:57.656812686Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-11T12:55:58.877804695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:55:58.886662432Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-11T12:55:59.268632543Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T12:56:00.460058663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:56:00.468916893Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:56:00.910352452Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-11T12:56:01.226171366Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:56:01.355618322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:56:01.359043908Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T12:56:01.752409056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:01.854728024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:56:01.858955042Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:56:03.81306293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:56:03.817196431Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:56:04.163520018Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-11T12:56:04.39716504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:56:04.401157407Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:56:06.752787358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:07.416302464Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T12:56:10.669366379Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T12:56:11.111821297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:56:11.115875406Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:56:11.730250225Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:56:11.730715238Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:56:11.752361858Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:56:11.752395384Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:56:11.752403904Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:56:11.752424167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:13.922298492Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-05-11T12:56:15.438346793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51288"
level=warn ts=2026-05-11T12:56:16.752892063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:17.174728362Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-11T12:56:20.428686779Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-11T12:56:21.671375636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:56:21.678315749Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:56:21.752557484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:22.260517689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:56:22.26534616Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:56:22.265371291Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:56:22.265400311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:56:22.270072265Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:56:23.681446003Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T12:56:23.681469661Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T12:56:23.931096108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:56:23.935440492Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:56:24.803959572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:56:24.807908731Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=warn ts=2026-05-11T12:56:26.752378861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:26.934673563Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-11T12:56:27.811787479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:56:27.815981623Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:56:29.192028646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:56:29.195505024Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-11T12:56:29.269988322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:56:30.142618519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:56:30.145938691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:56:30.188432246Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T12:56:30.188454561Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-05-11T12:56:30.891261094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:56:30.900878222Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:56:31.752325466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:31.941731379Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:56:32.504726553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:56:32.508331271Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:56:32.767780268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:56:32.771941939Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:56:33.43990871Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=debug ts=2026-05-11T12:56:36.693588269Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-11T12:56:36.752079583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:38.555625358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:56:38.563327189Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:56:39.947291577Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T12:56:41.752278972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:56:41.752318663Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T12:56:41.752324132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:41.752340225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T12:56:43.200035615Z caller=checkpoint.go:372 msg="writing series" size="987 kB"
level=debug ts=2026-05-11T12:56:43.200060689Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=4
level=info ts=2026-05-11T12:56:46.274490617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:56:46.278535727Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:56:46.454164393Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-11T12:56:46.454190381Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=1
level=warn ts=2026-05-11T12:56:46.752452287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:46.829809743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:56:46.838131579Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:56:47.673818058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:56:47.678468034Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T12:56:48.303795689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:56:48.307443634Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:56:48.307470964Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:56:48.307506572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:56:48.310663027Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:56:49.704797197Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-11T12:56:51.752396573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:56:52.926884091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:56:52.934917948Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:56:52.95800717Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-11T12:56:54.706050584Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T12:56:54.707722596Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T12:56:54.931889903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:56:54.939540253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:56:55.686795656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:56:55.690170975Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:56:56.21191414Z caller=checkpoint.go:372 msg="writing series" size="565 kB"
level=info ts=2026-05-11T12:56:56.364104901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:56:56.368253899Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T12:56:56.752083142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:56:57.377261457Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T12:56:59.2713464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T12:56:59.464871794Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=warn ts=2026-05-11T12:57:01.751870506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:02.717677525Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T12:57:03.94063072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:57:03.944744084Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:57:05.977004714Z caller=checkpoint.go:372 msg="writing series" size="4.3 MB"
level=debug ts=2026-05-11T12:57:05.977037176Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
level=info ts=2026-05-11T12:57:06.436858133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:57:06.441683011Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T12:57:06.452701102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:57:06.456927991Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:57:06.752726754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:09.223268253Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-11T12:57:10.918323644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:57:10.926879044Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:57:11.729940069Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:57:11.730983329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:57:11.752607249Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:57:11.752641633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:57:11.752651886Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:57:11.752734071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:12.47596248Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:57:14.083634719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:57:14.087348006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:57:14.627921583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:57:14.631420041Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-11T12:57:15.322695555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37198"
level=debug ts=2026-05-11T12:57:15.729672003Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-11T12:57:16.26117279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:57:16.265355222Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-11T12:57:16.265380116Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:57:16.265409267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:57:16.268834314Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T12:57:16.752466435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:57:17.349060491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:57:17.352489247Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:57:18.983047547Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-05-11T12:57:19.009044986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:57:19.012440553Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:57:20.32565513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:57:20.329330468Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T12:57:21.752401907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:22.224143236Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T12:57:22.235406056Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-05-11T12:57:25.488957263Z caller=checkpoint.go:372 msg="writing series" size="557 kB"
level=debug ts=2026-05-11T12:57:25.488984028Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-11T12:57:26.752136088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:57:26.842464001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:57:26.850096838Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:57:27.456108573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:57:27.464714124Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T12:57:27.743233384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:57:27.747267444Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:57:27.836506843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:57:27.839887588Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:57:28.708245337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:57:28.712287791Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:57:28.74186026Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
ts=2026-05-11T12:57:29.27267424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T12:57:30.198271683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:57:30.202030561Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:57:31.752381982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:32.007955852Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T12:57:32.007985316Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=debug ts=2026-05-11T12:57:35.248227082Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T12:57:36.314226048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:57:36.318647984Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:57:36.752850849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:38.500300032Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-11T12:57:40.627725885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:57:40.636371895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:57:40.636404863Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:57:40.636445018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:57:40.644908821Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:57:41.752843971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:57:41.75287406Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:57:41.752883249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:57:41.752899283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:57:41.752941743Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T12:57:41.757438503Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-11T12:57:41.760720495Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007817825s
level=info ts=2026-05-11T12:57:43.261638032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:57:43.271626651Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:57:44.200301757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:57:44.204871454Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:57:45.442881263Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:51640"
level=warn ts=2026-05-11T12:57:46.752490567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:57:47.950807412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:57:47.955800073Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:57:51.150430463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:57:51.160315952Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T12:57:51.752438319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:57:52.9516232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:57:52.955747089Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:57:54.132505229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:57:54.408264584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:57:54.415596545Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T12:57:54.640099114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T12:57:56.752350228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:57:58.383970335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:57:58.387959392Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-11T12:57:59.274166493Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=warn ts=2026-05-11T12:58:01.752759262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:02.570696983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:58:02.57419645Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:58:02.574226686Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:02.57425261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:58:02.57703789Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:58:03.388635212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:58:03.392817176Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T12:58:05.607434925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:58:05.611227047Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:58:05.611252076Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:05.611278923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:58:05.614051785Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:58:06.752357172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:08.703676383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:58:08.707185253Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:58:08.707211028Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:08.707236153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:58:08.710759004Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:58:11.25949472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:58:11.266063913Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T12:58:11.729508833Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:58:11.730543242Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T12:58:11.752044102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:58:11.752078359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:58:11.752087133Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T12:58:11.752093063Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-11T12:58:11.75212077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:11.75235409Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-11T12:58:11.752366299Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T12:58:11.752386301Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.21µs success=true
level=debug ts=2026-05-11T12:58:11.752637872Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T12:58:12.135155004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:58:12.138732038Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-11T12:58:12.34339757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:54088"
level=info ts=2026-05-11T12:58:12.817484021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:58:12.821049908Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T12:58:15.006972424Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
ts=2026-05-11T12:58:15.324981663Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46588"
level=info ts=2026-05-11T12:58:16.130223627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:58:16.134605391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T12:58:16.752345002Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:17.520253873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:58:17.523920284Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:58:18.259397162Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T12:58:20.912360434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:58:20.91634058Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:58:21.094709174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:58:21.098116178Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T12:58:21.512324733Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T12:58:21.752392584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:21.772661661Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:58:21.897893395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:58:21.905817071Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T12:58:22.233114774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:58:22.237017187Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:58:22.705906276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:58:22.714581577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:58:24.765305925Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-11T12:58:25.551387933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:58:25.556907551Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T12:58:26.752986849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:28.018206748Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-05-11T12:58:28.402186612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:58:28.406519197Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:58:29.27561613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:58:31.261713562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:58:31.269800084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T12:58:31.270924995Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=debug ts=2026-05-11T12:58:31.270939351Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-11T12:58:31.474720162Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T12:58:31.752870598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:32.909015487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:58:32.91335553Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T12:58:34.524284819Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-11T12:58:34.726643032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:58:34.731239911Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T12:58:35.258826459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:58:35.266240876Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:58:35.267199908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:58:35.270143826Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:58:35.270169181Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:35.27019678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:58:35.273175865Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T12:58:36.752386687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:37.777505451Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-11T12:58:41.030478401Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-11T12:58:41.752685411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:58:41.752715091Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:58:41.752723345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:58:41.752715541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:44.395877243Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T12:58:44.395924383Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=4
ts=2026-05-11T12:58:45.326881948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51116"
level=warn ts=2026-05-11T12:58:46.75284622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:47.53623041Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-11T12:58:50.620431711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:58:50.639697995Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:58:50.790673911Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-11T12:58:50.790711073Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-11T12:58:50.909070647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:58:50.917598399Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-11T12:58:50.917643873Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:50.917687407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:58:50.926130411Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:58:51.75190932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:58:52.162221749Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:58:53.304610023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:58:53.316113602Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:58:53.316155238Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:53.316198662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:58:53.327190491Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:58:53.411702982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:58:53.425586865Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:58:53.465429099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:58:53.476249187Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:58:54.075383733Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T12:58:54.07543081Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=warn ts=2026-05-11T12:58:56.75240285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:58:56.895914429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:58:56.910110944Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T12:58:57.295689409Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T12:58:58.030884533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:58:58.04054613Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:58:58.040604323Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:58.040646254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:58:58.050961201Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-11T12:58:59.277220726Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T12:58:59.508736032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:58:59.518029721Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:58:59.518095999Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:58:59.518137759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:58:59.536070652Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T12:59:00.123670148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:59:00.134937965Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T12:59:00.548142264Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T12:59:01.751898881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:03.748640011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:59:03.767881011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:59:03.802413991Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T12:59:04.059481712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:59:04.08091619Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:59:05.298229151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:59:05.307450561Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T12:59:06.752368363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:07.054893275Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=info ts=2026-05-11T12:59:08.651462319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:59:08.655223391Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:59:10.310218227Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T12:59:10.310246231Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-05-11T12:59:11.177582807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:59:11.181308088Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:59:11.181339732Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:59:11.181367906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:59:11.184922982Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T12:59:11.730293351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T12:59:11.730309069Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T12:59:11.75246536Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:59:11.752500506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:59:11.752509175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:59:11.752610621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:12.80013299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:59:12.80390722Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T12:59:13.081703034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T12:59:13.085251393Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T12:59:13.561234883Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=info ts=2026-05-11T12:59:14.652073101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:59:14.656030926Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-11T12:59:15.328581812Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33542"
ts=2026-05-11T12:59:15.447787147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:42412"
level=warn ts=2026-05-11T12:59:16.752253965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:16.813439014Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-11T12:59:17.549509275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:59:17.557962546Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:59:17.557999955Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:59:17.558033415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:59:17.566072715Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T12:59:18.180961574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T12:59:18.189269067Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T12:59:18.559081369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:59:18.563507449Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:59:18.563533182Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T12:59:18.563558666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:59:18.567318802Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T12:59:19.368702081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T12:59:19.376945545Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T12:59:20.066061111Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=info ts=2026-05-11T12:59:21.27553513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T12:59:21.28033571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T12:59:21.752693213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:23.320414558Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T12:59:23.320438022Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-11T12:59:24.301697065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:59:24.304973669Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:59:26.57328336Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=warn ts=2026-05-11T12:59:26.752317698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:28.04029301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T12:59:28.040367378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="175 kB" avg_comp="175 kB" total_uncomp="908 kB" avg_uncomp="908 kB" idle=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T12:59:28.04464033Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T12:59:28.340578375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T12:59:28.344265799Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T12:59:29.088252443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T12:59:29.092865853Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T12:59:29.281859807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T12:59:29.825801838Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=info ts=2026-05-11T12:59:30.400156824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T12:59:30.408645663Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T12:59:31.752677808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:31.877004663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T12:59:31.880645029Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T12:59:32.494244903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T12:59:33.048197911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T12:59:33.053557125Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T12:59:33.078736114Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T12:59:33.452909549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T12:59:33.456454134Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T12:59:35.188035645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T12:59:35.19161208Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T12:59:36.331721127Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-11T12:59:36.752376315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:39.584179558Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-05-11T12:59:41.366957546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T12:59:41.371233926Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T12:59:41.752315077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T12:59:41.752344875Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T12:59:41.752354722Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T12:59:41.752410359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:42.251521959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T12:59:42.255820776Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T12:59:42.553984325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T12:59:42.557737792Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T12:59:42.671118202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T12:59:42.675542721Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T12:59:42.837656921Z caller=checkpoint.go:372 msg="writing series" size="319 kB"
level=debug ts=2026-05-11T12:59:42.837680234Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-11T12:59:43.216333119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T12:59:43.220265691Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-11T12:59:45.330272167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55220"
level=debug ts=2026-05-11T12:59:46.09110324Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-11T12:59:46.752306514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:48.15619171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T12:59:48.160088756Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T12:59:48.333627074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T12:59:48.337079195Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T12:59:49.343450979Z caller=checkpoint.go:372 msg="writing series" size="515 kB"
level=warn ts=2026-05-11T12:59:51.752736784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T12:59:52.596722133Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T12:59:53.046468313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T12:59:53.050179592Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T12:59:53.528707988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T12:59:53.53520867Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T12:59:55.549410547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T12:59:55.557020702Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T12:59:55.849645027Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T12:59:56.752773087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T12:59:58.60603607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T12:59:58.609702334Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T12:59:59.102293451Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T12:59:59.283162746Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:00:01.530316922Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T13:00:01.752609434Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:02.343174101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:00:02.353283669Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:00:02.355686382Z caller=checkpoint.go:372 msg="writing series" size="588 kB"
level=debug ts=2026-05-11T13:00:02.355705442Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-11T13:00:03.28508086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:00:03.29039088Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:00:03.691699425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:00:03.695691806Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:00:04.548600262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:00:04.552833549Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:00:04.793639557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:00:04.800620488Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T13:00:05.609496116Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=info ts=2026-05-11T13:00:05.919082568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:00:05.92787551Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:00:06.752586787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:08.605883829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:00:08.609393495Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:00:08.861959839Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-11T13:00:11.729533388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:00:11.730588222Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T13:00:11.731918932Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T13:00:11.73195958Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976115 n=1
level=debug ts=2026-05-11T13:00:11.732001484Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976115 n=1
level=info ts=2026-05-11T13:00:11.751833215Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:00:11.751862923Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:00:11.751871239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:00:11.751867462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:00:12.114452924Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-11T13:00:15.369894827Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T13:00:15.369921276Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-11T13:00:16.752789252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:17.059557161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:00:17.063906616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:00:17.862441866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:00:17.867002547Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:00:18.620433296Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T13:00:20.973170053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:00:20.977430784Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:00:21.258209438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:00:21.261720459Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:00:21.752511658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:00:21.873827678Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-11T13:00:25.146205398Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T13:00:25.146246614Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=warn ts=2026-05-11T13:00:26.751958521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:00:27.898972177Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:00:28.379600392Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T13:00:29.085692782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:00:29.089714756Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T13:00:29.284673484Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T13:00:30.008750904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:00:31.633496409Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=info ts=2026-05-11T13:00:31.698282249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:00:31.702129587Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T13:00:31.752552567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:31.767695912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:00:31.771368016Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:00:34.886180093Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-11T13:00:36.415606185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:00:36.419987104Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:00:36.695712738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:00:36.704309102Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:00:36.752262151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:00:38.138466656Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=info ts=2026-05-11T13:00:40.834714156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:00:40.83898084Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:00:41.392192135Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-11T13:00:41.752814767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:00:41.752847636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:00:41.752858236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:00:41.752874839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:00:42.35545488Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:36976"
level=info ts=2026-05-11T13:00:42.603439391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:00:42.607327249Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T13:00:44.64545477Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-11T13:00:44.965631181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:00:44.969641845Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
ts=2026-05-11T13:00:45.333057599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44360"
level=warn ts=2026-05-11T13:00:46.752023621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:47.016237773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:00:47.019981699Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:00:47.898060692Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T13:00:50.9942802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:00:50.997798651Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:00:51.150976488Z caller=checkpoint.go:372 msg="writing series" size="714 kB"
level=debug ts=2026-05-11T13:00:51.151004852Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-11T13:00:51.728929924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:00:51.732374226Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-11T13:00:51.752665211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:00:53.273379075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:00:53.281930583Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:00:53.893840139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:00:53.898832164Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:00:54.259989839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:00:54.270518236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:00:54.404244286Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-11T13:00:55.318010395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:00:55.326258257Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:00:55.326287876Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:00:55.326335755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:00:55.334636902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:00:56.7529815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:00:57.656742754Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-11T13:00:58.636933379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:00:58.643290375Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:00:58.643322873Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:00:58.643352135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:00:58.647079732Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-11T13:00:59.286403269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T13:00:59.508788024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:00:59.511981548Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:01:00.409837162Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:01:00.409890895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="81 kB" avg_comp="81 kB" total_uncomp="412 kB" avg_uncomp="412 kB" idle=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:01:00.413583727Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=debug ts=2026-05-11T13:01:00.910056131Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-11T13:01:01.406931751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:01:01.410352442Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T13:01:01.752706907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:03.220348346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:01:03.227881093Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T13:01:04.162912934Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-11T13:01:05.94653318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:01:05.950037813Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:01:06.752100409Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:01:07.415591432Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-05-11T13:01:10.669083504Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-11T13:01:11.729855476Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:01:11.730916434Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:01:11.751798401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:01:11.751830741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:01:11.751839444Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:01:11.751873783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:01:12.356811622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:35922"
level=info ts=2026-05-11T13:01:13.206877174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:01:13.211397075Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=debug ts=2026-05-11T13:01:13.921611892Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T13:01:15.261033466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:01:15.269484871Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:01:15.658624536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:01:15.662512222Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:01:16.669627845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:01:16.673908205Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T13:01:16.752060207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:01:17.174854359Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-11T13:01:17.854049254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:01:17.863869148Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:01:17.863898748Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:01:17.863927081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:01:17.871327039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T13:01:20.427377633Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-11T13:01:21.752277291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:01:23.681723561Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T13:01:23.681746504Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-05-11T13:01:25.432632913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:01:25.436654911Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:01:25.541028397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:01:25.544742454Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:01:26.404473017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:01:26.406132959Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-11T13:01:26.752449078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:01:26.933878517Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-11T13:01:28.981696667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:01:28.98601342Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:01:29.101263694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:01:29.104695849Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
ts=2026-05-11T13:01:29.287561311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:01:30.187740227Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T13:01:30.187764963Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=debug ts=2026-05-11T13:01:30.929601926Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:01:30.929602002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="67 kB" avg_uncomp="67 kB" idle=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:01:30.933132029Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=warn ts=2026-05-11T13:01:31.752542119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:32.411078124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:01:32.418537811Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:01:32.427626359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:01:32.431807536Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:01:32.806980453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:01:32.811361765Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:01:33.396976632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:01:33.40046535Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:01:33.43971626Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T13:01:35.186472416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:01:35.190005498Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=info ts=2026-05-11T13:01:36.266301886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:01:36.270202149Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T13:01:36.692509607Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-11T13:01:36.752237011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:39.79169475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:01:39.795629499Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:01:39.946767232Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T13:01:41.7527559Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:01:41.7527893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:01:41.752797868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:01:41.752852226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:42.132368811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:01:42.136010309Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:01:43.200145445Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-11T13:01:43.200169508Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=4
level=info ts=2026-05-11T13:01:44.521109025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:01:44.529646986Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:01:44.529673729Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:01:44.529701045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:01:44.535309724Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T13:01:45.336218476Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35778"
level=debug ts=2026-05-11T13:01:46.45444998Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T13:01:46.454481471Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=1
level=warn ts=2026-05-11T13:01:46.752493653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:46.82677345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:01:46.835667958Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:01:46.951542837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:01:46.960559602Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T13:01:49.663908414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:01:49.667979078Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:01:49.705079962Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-11T13:01:50.1788856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:01:50.182759764Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T13:01:51.752048843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:01:52.361624117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:01:52.369716941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:01:52.821140134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:01:52.824956197Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:01:52.958419778Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-11T13:01:54.114187421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:01:54.119314941Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:01:56.211604039Z caller=checkpoint.go:372 msg="writing series" size="628 kB"
level=warn ts=2026-05-11T13:01:56.752326894Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:01:59.289171322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:01:59.464680466Z caller=checkpoint.go:372 msg="writing series" size="145 kB"
level=info ts=2026-05-11T13:01:59.720411101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:01:59.726163217Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:02:00.240642405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:02:00.248215001Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:02:00.269276741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:02:00.272904073Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:02:01.029346107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:02:01.035954577Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-11T13:02:01.752111562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:02:02.407737113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:02:02.717217872Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-11T13:02:05.97519704Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-05-11T13:02:05.975220785Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.2 MB" series=6
level=info ts=2026-05-11T13:02:06.421485065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:02:06.427041568Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T13:02:06.75285816Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:08.057118298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:02:08.061332286Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:02:09.226051654Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-11T13:02:09.578699622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:02:09.582163875Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T13:02:11.729861489Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:02:11.73092345Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:02:11.752535472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:02:11.752580633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T13:02:11.75258601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:11.75260036Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T13:02:12.312925501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:02:12.316479024Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:02:12.475683088Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-11T13:02:15.337462657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41220"
level=info ts=2026-05-11T13:02:15.371127414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:02:15.37468429Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=debug ts=2026-05-11T13:02:15.728976185Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-05-11T13:02:16.75235806Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:18.665504848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:02:18.669386455Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:02:18.982692228Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=debug ts=2026-05-11T13:02:20.024354154Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T13:02:21.752712325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:02:22.235689268Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-11T13:02:24.048067103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:02:24.05174972Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:02:24.361622895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:02:24.365097489Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:02:24.667872698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:02:24.671311502Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:02:24.866111076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:02:24.869722122Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:02:25.488649638Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-11T13:02:25.488671585Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-11T13:02:26.752393557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:27.501509367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:02:27.509190804Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:02:27.959929886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:02:27.963436628Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:02:28.741361843Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-11T13:02:29.291041062Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=info ts=2026-05-11T13:02:31.031763298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:02:31.038445687Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-11T13:02:31.751863272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:02:32.0109475Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T13:02:32.010988989Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
level=info ts=2026-05-11T13:02:32.012458923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="438 kB" avg_comp="438 kB" total_uncomp="2.2 MB" avg_uncomp="2.2 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:02:32.01658499Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:02:34.775016695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:02:34.779606035Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:02:34.875796021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:02:34.883868138Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:02:35.247756044Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=warn ts=2026-05-11T13:02:36.752677397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:02:37.004213979Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:02:38.500407713Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-11T13:02:41.75206058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:41.752066969Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:02:41.752091557Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:02:41.752101028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T13:02:41.753193432Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T13:02:41.760478331Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-11T13:02:41.762220412Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009671573s
level=info ts=2026-05-11T13:02:42.736773142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:02:42.74225324Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:02:44.568633342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:02:44.572537542Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:02:45.277291103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:02:45.286323995Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:02:46.482391364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:02:46.485749788Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T13:02:46.75227083Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:48.007975611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:02:48.012593344Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:02:49.866847566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:02:49.870524029Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T13:02:51.752481165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:54.381627932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:02:54.385348144Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:02:54.601868223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:02:54.61003707Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:02:56.751963641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:02:58.224223155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:02:58.227843039Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-11T13:02:59.292368414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:03:00.621085997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:03:00.62707226Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:03:01.580165556Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T13:03:01.75186729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:03:01.811111491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:03:01.814985407Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:03:01.998444953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:03:02.002372146Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=warn ts=2026-05-11T13:03:06.752379988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:08.276786897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:03:08.613692461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:03:08.618100507Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:03:09.740695776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:03:09.745696384Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:03:11.696650084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:03:11.70000162Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:03:11.729933228Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:03:11.730990285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:03:11.752385438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-11T13:03:11.752410615Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-05-11T13:03:11.752425513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:03:11.752435811Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:03:11.752466895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:03:11.752594818Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-11T13:03:11.752603755Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T13:03:11.752623176Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.721µs success=true
level=debug ts=2026-05-11T13:03:11.752838192Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-11T13:03:12.363655555Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39612"
level=info ts=2026-05-11T13:03:14.344859521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:03:14.350010276Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:03:14.899542896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:03:14.907837036Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:03:15.006503995Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-05-11T13:03:16.75268282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:18.259420869Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T13:03:18.675335749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:03:18.679367902Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:03:18.679392301Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:03:18.679428871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:03:18.682921934Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:03:19.386196898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:03:19.389716985Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:03:19.772032775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:03:19.776070173Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:03:20.128869495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:03:20.137604194Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:03:21.195176828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:03:21.198745966Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:03:21.512561383Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T13:03:21.752699158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:24.766338903Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-05-11T13:03:26.752230965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:28.018529996Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-11T13:03:28.018552526Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-11T13:03:28.959422331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:03:28.963512822Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
ts=2026-05-11T13:03:29.293730125Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:03:29.298449591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:03:29.305235969Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T13:03:31.271291641Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-11T13:03:31.752879411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:03:32.091537046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:03:32.122042753Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:03:33.543689593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:03:33.547665248Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:03:33.632888872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:03:33.636292032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:03:34.524351226Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-11T13:03:35.671289077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:03:35.675583437Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T13:03:36.752711699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:37.214237548Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:03:37.77786835Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-05-11T13:03:40.06001957Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:03:40.06695534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:03:40.07507848Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:03:40.493260022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:03:40.499041429Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:03:41.030922368Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-11T13:03:41.752098982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:03:41.752133613Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T13:03:41.752120362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:03:41.752142574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T13:03:42.350326986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:03:42.358627751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T13:03:44.395910713Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T13:03:44.3959531Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=5
ts=2026-05-11T13:03:45.342027413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56216"
level=warn ts=2026-05-11T13:03:46.752328616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:47.536428593Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-11T13:03:49.7335254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:03:49.745172035Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:03:49.745215712Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:03:49.745257453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:03:49.755761728Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:03:49.979518161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:03:49.992674765Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-11T13:03:49.99271221Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:03:50.046338497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:03:50.062046792Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:03:50.791294244Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T13:03:50.793254926Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-11T13:03:51.751946244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:03:52.0575188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:03:52.066195596Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=info ts=2026-05-11T13:03:52.39245697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:03:52.401830967Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:03:53.525000245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:03:53.53345018Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:03:53.814585494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:03:53.826268611Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:03:54.078749027Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-11T13:03:54.078790624Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=warn ts=2026-05-11T13:03:56.752235972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:03:57.296298325Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T13:03:57.302142351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:03:57.320303151Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
ts=2026-05-11T13:03:59.295512243Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T13:04:00.548419442Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T13:04:01.751894252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:04:03.270114933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:04:03.288909313Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:04:03.802294522Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T13:04:04.973062797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:04:04.987695647Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:04:05.583364367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:04:05.597667804Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:04:06.339542645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:04:06.352777586Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:04:06.493368842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:04:06.50295134Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:04:06.703033121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:04:06.727154671Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:04:06.752310403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:07.055139837Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=info ts=2026-05-11T13:04:07.457912674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:04:07.473696786Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:04:09.115667993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:04:09.128633865Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:04:09.33049844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:04:09.341928575Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:04:10.124405739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:04:10.145625616Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:04:10.314120707Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T13:04:10.31607496Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=debug ts=2026-05-11T13:04:11.729614335Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:04:11.730661659Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:04:11.752440239Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:04:11.75247345Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:04:11.752482744Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:04:11.752520358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:12.351400652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:04:12.351474019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="816 B" avg_comp="816 B" total_uncomp="65 kB" avg_uncomp="65 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:04:12.356007231Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-11T13:04:13.561059815Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=warn ts=2026-05-11T13:04:16.752177998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:16.813373293Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-11T13:04:19.213337157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:04:19.216917306Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:04:19.365386921Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:04:20.066446553Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-11T13:04:20.76446636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:04:20.768537562Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T13:04:21.752782124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:23.320739284Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T13:04:23.320762979Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-11T13:04:24.657228093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:04:24.664301824Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:04:26.391106565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:04:26.394665804Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:04:26.572433503Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=info ts=2026-05-11T13:04:26.644424696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:04:26.652371161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:04:26.652400622Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:04:26.652432428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:04:26.659840889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T13:04:26.752347866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:04:26.95045164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:04:26.954136775Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:04:26.954172577Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:04:26.954206583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:04:26.957192723Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-11T13:04:29.299423769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=debug ts=2026-05-11T13:04:29.825434145Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-05-11T13:04:31.751968414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:04:32.259062609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:04:32.263396816Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:04:32.506530781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:04:32.510984909Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:04:32.565246096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:04:32.568796492Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=debug ts=2026-05-11T13:04:33.078407943Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T13:04:34.558256712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:04:34.562737301Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:04:36.331421336Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-11T13:04:36.752544547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:39.584945049Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=info ts=2026-05-11T13:04:39.661085453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:04:39.670363607Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:04:39.946440289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:04:39.956760794Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:04:40.637626556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:04:40.641754309Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:04:41.412063865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:04:41.416038303Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:04:41.416062229Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:04:41.416086659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:04:41.418970664Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:04:41.752208834Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:04:41.752242564Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:04:41.752250797Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:04:41.752292035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:42.837950328Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-05-11T13:04:43.283153066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:04:43.286811266Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T13:04:44.502641959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:04:44.808629805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:04:44.812504988Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:04:46.090689933Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-05-11T13:04:46.090713718Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-11T13:04:46.752627229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:04:48.675581615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:04:48.680056937Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:04:49.344537338Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
level=info ts=2026-05-11T13:04:49.350352554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:04:49.354098317Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=warn ts=2026-05-11T13:04:51.752544598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:04:51.796704669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:04:51.801529199Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:04:51.801554743Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:04:51.801598166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:04:51.804609623Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:04:52.597254062Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T13:04:52.641367987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:04:52.645272842Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:04:52.6452998Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:04:52.645326262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:04:52.648477601Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:04:52.717690379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:04:52.721148744Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:04:55.414901435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:04:55.423464755Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:04:55.849396588Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T13:04:56.606037411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:04:56.610099043Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-11T13:04:56.752235005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:04:59.102527867Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T13:04:59.300449627Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T13:05:00.981419441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:05:00.98503493Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:05:01.190624535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:05:01.194211379Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T13:05:01.752359277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:02.336349769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:05:02.340297277Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:05:02.355918163Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-05-11T13:05:02.355941649Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-11T13:05:02.888009912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:05:02.892491367Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:05:03.104110762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:05:03.108346559Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:05:04.512960527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:05:04.520487089Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:05:05.267527044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:05:05.271454189Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:05:05.609386832Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=warn ts=2026-05-11T13:05:06.752424325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:07.273390838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:05:07.279600459Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:05:08.084430626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:05:08.088024269Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-11T13:05:08.861649406Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T13:05:08.958963487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:05:08.96691609Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:05:11.729893979Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:05:11.730937326Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:05:11.752047534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:05:11.752079602Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:05:11.752087889Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:05:11.752098867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:05:12.115254569Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=info ts=2026-05-11T13:05:13.375628967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:05:13.379648656Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:05:15.369688837Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T13:05:15.369725354Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=4
level=warn ts=2026-05-11T13:05:16.75295472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:17.184620468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:05:17.188234781Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:05:17.188261228Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:17.18828716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:05:17.191292949Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:05:17.902953519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:05:17.904763292Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-11T13:05:18.262007342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:05:18.265808232Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:05:18.620913483Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-11T13:05:18.871988411Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:05:20.134465772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:05:20.139026619Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T13:05:21.752838153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:05:21.873878919Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T13:05:24.06042543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:05:24.064275596Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:05:25.147060098Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T13:05:25.147112031Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T13:05:25.851284271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:05:25.855167043Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=info ts=2026-05-11T13:05:26.380130968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:05:26.388678039Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:05:26.752501557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:05:28.37988326Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T13:05:28.434314594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:05:28.439640258Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T13:05:29.302544239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:05:30.507032348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:05:30.510327739Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:05:30.868686537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:05:30.872444735Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:05:31.633271984Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-11T13:05:31.752862764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:33.46980199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:05:33.473262493Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:05:33.564618605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:05:33.568005398Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:05:34.886351393Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=info ts=2026-05-11T13:05:36.603859165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:05:36.609064909Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T13:05:36.752425983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:38.053367398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:05:38.056892812Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:05:38.056919128Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:38.056946106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:05:38.059976324Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:05:38.139420844Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-11T13:05:41.056518857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:05:41.060847349Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-11T13:05:41.060871297Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:41.060897233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:05:41.063715302Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:05:41.391711352Z caller=checkpoint.go:372 msg="writing series" size="5.6 kB"
level=info ts=2026-05-11T13:05:41.751898916Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:05:41.751933846Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:05:41.751942226Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:05:41.751949143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:41.9515296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:05:41.961049216Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:05:43.397609467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:05:43.401223708Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:05:43.401254235Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:43.401281557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:05:43.404273574Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:05:43.580879083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:05:43.584256738Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:05:44.644945878Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-11T13:05:46.157888473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:05:46.162382065Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:05:46.162406811Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:46.162432935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:05:46.16597881Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T13:05:46.752702777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:47.169869391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:05:47.174369213Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:05:47.174401702Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:47.174426489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:05:47.178011073Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:05:47.898041452Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T13:05:49.546699178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:05:49.550409614Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:05:51.1518309Z caller=checkpoint.go:372 msg="writing series" size="730 kB"
level=debug ts=2026-05-11T13:05:51.151855262Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-11T13:05:51.443178094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-11T13:05:51.752688112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:53.72697956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:05:53.730460912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T13:05:54.404605091Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-11T13:05:54.908208717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:05:54.913364309Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:05:56.051363432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:05:56.060595624Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:05:56.060635223Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:56.060665738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:05:56.069026745Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T13:05:56.752788025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:05:56.903101103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:05:56.906835781Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:05:57.209936239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:05:57.213462891Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:05:57.213488595Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:05:57.21351537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:05:57.216800079Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:05:57.656487025Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-11T13:05:57.874323545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:05:57.878170577Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
ts=2026-05-11T13:05:59.303946433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:05:59.84747729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:05:59.85580153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:06:00.885039857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:06:00.888775435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:06:00.909875783Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T13:06:01.752138909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:03.843869022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:06:03.848184905Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:06:04.163250041Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-11T13:06:06.752866477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:06.779055294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:06:06.787160426Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:06:06.882386442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:06:06.891319256Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:06:07.295637115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:06:07.299955954Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:06:07.416071309Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-05-11T13:06:09.424578997Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T13:06:09.720364358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:06:09.723990829Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:06:09.724015088Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:06:09.724039669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:06:09.727191995Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:06:10.367160005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:06:10.370645965Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:06:10.539241855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:06:10.547338302Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:06:10.669190782Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T13:06:10.980217547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:06:10.983699714Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=debug ts=2026-05-11T13:06:11.730169528Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:06:11.731209862Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:06:11.751959216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T13:06:11.751969534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:06:11.751989985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:06:11.751998455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T13:06:13.922669928Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T13:06:14.614586615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:06:14.618333767Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=warn ts=2026-05-11T13:06:16.752009841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:06:17.175107254Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=debug ts=2026-05-11T13:06:17.719472163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:06:18.684259282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:06:18.688991525Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:06:19.33823208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:06:19.341739396Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:06:20.428306155Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-11T13:06:20.706050674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:06:20.710342836Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:06:20.71848116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:06:20.722124032Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:06:21.439224833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:06:21.444238208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T13:06:21.75238137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:22.248555359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:06:22.252084087Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:06:22.612902525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:06:22.616308114Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:06:23.529294288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:06:23.532838157Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:06:23.681660678Z caller=checkpoint.go:372 msg="writing series" size="775 kB"
level=debug ts=2026-05-11T13:06:23.681684478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-11T13:06:24.346656696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:06:24.353877823Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:06:26.752660208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:26.860985567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="843 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:06:26.865798822Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:06:26.934152283Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-11T13:06:28.397010822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:06:28.400549729Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-11T13:06:29.305731739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:06:30.188016113Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T13:06:30.1880393Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-11T13:06:31.752729631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:32.474695973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:06:32.480128781Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:06:33.439923587Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T13:06:33.453056627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:06:33.45664545Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:06:34.620134648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:06:34.6276507Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:06:36.693206265Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-11T13:06:36.752257967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:38.412057594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:06:38.416409084Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:06:39.947041272Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-11T13:06:40.061954463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:06:40.065224804Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:06:41.415273659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:06:41.41880375Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:06:41.752094481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:06:41.752122723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:06:41.75213359Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:06:41.752156973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:06:42.373946434Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:53866"
level=debug ts=2026-05-11T13:06:43.200220532Z caller=checkpoint.go:372 msg="writing series" size="860 kB"
level=debug ts=2026-05-11T13:06:43.200245162Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-05-11T13:06:43.719976061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:06:43.723463884Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:06:44.750129633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-11T13:06:45.352500843Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53938"
level=debug ts=2026-05-11T13:06:45.396085363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:06:46.455640249Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-11T13:06:46.455671144Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=1
level=warn ts=2026-05-11T13:06:46.752501893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:47.028850404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:06:47.037415967Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:06:49.516606752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:06:49.524417241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T13:06:49.704664832Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-11T13:06:51.752091834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:52.604584488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3cfc8856376e0ec2 immediate=false num_chunks=1 total_comp="98 kB" avg_comp="98 kB" total_uncomp="412 kB" avg_uncomp="412 kB" full=1 labels="{stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:06:52.608941405Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3cfc8856376e0ec2/19e16dd810f%3A19e16e93529%3A24fdf58b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-wlcf\"}"
level=error ts=2026-05-11T13:06:52.608968478Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:06:52.608997061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:06:52.613417866Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:06:52.957858972Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-11T13:06:55.123099872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:06:55.128558198Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:06:56.211406212Z caller=checkpoint.go:372 msg="writing series" size="511 kB"
level=warn ts=2026-05-11T13:06:56.752508897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:06:56.962378953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:06:56.965892519Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
ts=2026-05-11T13:06:59.307193394Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:06:59.464538405Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-11T13:07:01.013229793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:07:01.017004708Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:07:01.752316454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:02.7168465Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T13:07:03.831516352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:07:03.83828153Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:07:04.577217732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:07:04.581187008Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:07:05.976058039Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-11T13:07:05.976084821Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=6
level=warn ts=2026-05-11T13:07:06.752492997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:07:08.260688103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:07:08.265153463Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:07:08.592954389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:07:08.597415026Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:07:08.753218825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:07:08.756706544Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:07:09.223231607Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-05-11T13:07:09.944254136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:07:09.948732265Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:07:10.267468854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:07:10.271194601Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:07:11.729941272Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:07:11.730992784Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:07:11.751864376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:07:11.751900498Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:07:11.751909526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:07:11.751925188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:12.476720212Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-11T13:07:14.550919463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:07:14.556201617Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:07:14.579388623Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:07:14.62950005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:07:14.633108002Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:07:15.341455755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:07:15.34471699Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-11T13:07:15.471263356Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:36100"
level=debug ts=2026-05-11T13:07:15.729776421Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T13:07:15.741992812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:07:15.745545619Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:07:16.392188018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:07:16.39616175Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=warn ts=2026-05-11T13:07:16.7520202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:18.982743761Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-11T13:07:19.189233319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:07:19.192662677Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T13:07:21.752849795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:22.235679102Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-11T13:07:23.820082762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:07:23.828869418Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:07:23.828893678Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:07:23.828939658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:07:23.836961179Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:07:24.680087132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:07:24.683900345Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:07:25.488554868Z caller=checkpoint.go:372 msg="writing series" size="621 kB"
level=debug ts=2026-05-11T13:07:25.488596393Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=info ts=2026-05-11T13:07:25.914577501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:07:25.92303351Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T13:07:26.679808653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:07:26.683668012Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-11T13:07:26.752229086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:28.74155158Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-11T13:07:29.308443127Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:07:30.418855765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:07:30.42280864Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:07:31.105839254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:07:31.109640889Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:07:31.458167867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T13:07:31.752185546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:32.012945688Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-11T13:07:32.012983522Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=info ts=2026-05-11T13:07:32.309844512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:07:32.318071563Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:07:34.477170022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:07:34.481852594Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:07:35.205835927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:07:35.209515055Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:07:35.248346427Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T13:07:35.942167655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:07:35.951181681Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:07:36.752130454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:38.500679269Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-11T13:07:41.66869912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:07:41.676374266Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:07:41.676401824Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:07:41.676430185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:07:41.686365926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:07:41.752321535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:07:41.752359409Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:07:41.752368973Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:07:41.752416976Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:07:41.753475684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T13:07:41.760995814Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-11T13:07:41.762594513Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009824886s
level=info ts=2026-05-11T13:07:44.279996558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:07:44.283899155Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T13:07:44.82086766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T13:07:45.472940821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:52250"
level=warn ts=2026-05-11T13:07:46.751929191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:07:49.391164973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:07:49.395045661Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:07:51.751859143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:07:53.657993216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:07:53.664647117Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:07:55.237438559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:07:55.242430441Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T13:07:56.752725108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:07:57.453641194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:07:57.463381626Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:07:58.463354824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:07:58.468140147Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:07:59.09026974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:07:59.094661177Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:07:59.155992516Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:07:59.160213682Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:07:59.160241311Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:07:59.160268524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:07:59.164091314Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-11T13:07:59.309673982Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:08:00.5392481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:08:00.542878496Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:08:01.733251704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:08:01.737160476Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T13:08:01.752291359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:08:01.810485136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:08:01.814332398Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:08:01.81435877Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:08:01.814386336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:08:01.817505796Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:08:02.938624133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:08:02.942366854Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:08:04.236823844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:08:04.244894247Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-11T13:08:04.484424786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:08:04.487817579Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:08:06.104610351Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T13:08:06.752366436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:08:09.543642212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:08:09.551413793Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T13:08:11.730207341Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T13:08:11.730457912Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:08:11.752031334Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T13:08:11.752031036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T13:08:11.752053961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:11.752063958Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:08:11.752074024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T13:08:11.752198741Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-11T13:08:11.752206885Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T13:08:11.75222438Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.694µs success=true
level=debug ts=2026-05-11T13:08:11.752452658Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-11T13:08:12.369835571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:08:12.506049632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:08:12.511098387Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:08:15.005899589Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
ts=2026-05-11T13:08:15.474254845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:45940"
level=info ts=2026-05-11T13:08:16.291615604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:08:16.295749061Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T13:08:16.752841854Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:18.259840215Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T13:08:18.693822074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:08:18.698099237Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:08:18.964167849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:08:18.968280571Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:08:20.774179387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:08:20.778250581Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:08:21.512157761Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-11T13:08:21.751944401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:08:22.528618951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:08:22.532060731Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:08:23.035787963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:08:23.039473285Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-11T13:08:24.764944727Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-11T13:08:26.215987166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:08:26.219705617Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T13:08:26.752173091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:08:27.820009529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:08:27.828261852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T13:08:28.017938289Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-11T13:08:28.017962563Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-11T13:08:28.072947778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:08:28.077710842Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-05-11T13:08:29.311391741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T13:08:30.087688087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:08:30.090104196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:08:30.092438672Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:08:30.09243793Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:08:31.142498705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:08:31.147067536Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:08:31.271407553Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-11T13:08:31.307453399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:08:31.315144792Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:08:31.752183677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:34.523882351Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-11T13:08:36.75240125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:37.777723939Z caller=checkpoint.go:372 msg="writing series" size="177 kB"
level=info ts=2026-05-11T13:08:38.802517475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:08:38.8064025Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:08:40.171607405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:08:40.176603788Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:08:41.029923024Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-11T13:08:41.751903462Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T13:08:41.75192461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:41.751945411Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:08:41.751954889Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T13:08:41.972781002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:08:44.381479817Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T13:08:44.381520712Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=5
level=info ts=2026-05-11T13:08:45.449886724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:08:45.475794786Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:08:45.475837568Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:08:45.476250295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T13:08:45.476329744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:32808"
level=error ts=2026-05-11T13:08:45.495722772Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:08:46.626909405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:08:46.639901025Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:08:46.752305368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:47.535929357Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-11T13:08:50.600181698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:08:50.609278331Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:08:50.790726178Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T13:08:50.790764891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-11T13:08:51.332839065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:08:51.347801578Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T13:08:51.751886653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:08:53.63208402Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:08:53.640542146Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T13:08:54.073725182Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-11T13:08:54.073813602Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-11T13:08:55.863968897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:08:55.881320831Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:08:56.752350159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:08:57.054402914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:08:57.065650906Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:08:57.28135197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:08:57.293345588Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:08:57.296854269Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T13:08:57.335459901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:08:57.354280633Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:08:58.056841996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:08:58.070754352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-11T13:08:59.313909893Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:09:00.548007006Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-05-11T13:09:01.161175102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:09:01.17256083Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-11T13:09:01.7519344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:09:02.420507166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:09:02.438643337Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:09:03.801443206Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-11T13:09:06.752252869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:07.05472728Z caller=checkpoint.go:372 msg="writing series" size="316 kB"
level=debug ts=2026-05-11T13:09:10.313798241Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T13:09:10.313840726Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=5
level=debug ts=2026-05-11T13:09:11.730071958Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:09:11.731152823Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:09:11.75195774Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:09:11.751994349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:09:11.752004869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:09:11.752032576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:09:12.054486875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:09:12.05944723Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:09:12.33060627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:09:12.334481673Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-11T13:09:12.381267164Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:34456"
level=debug ts=2026-05-11T13:09:12.521538681Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:09:13.149368023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:09:13.153397708Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:09:13.415066461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:09:13.419154811Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:09:13.561146597Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
ts=2026-05-11T13:09:15.478316103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:50190"
level=info ts=2026-05-11T13:09:16.429045264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:09:16.435179525Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:09:16.485268722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:09:16.488861494Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:09:16.515937045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:09:16.519588186Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:09:16.605806242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:09:16.609117724Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:09:16.699376424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:09:16.702739309Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-11T13:09:16.752711838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:16.813426387Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-11T13:09:18.800705307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:09:18.808936894Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:09:19.583230572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:09:19.589979995Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T13:09:20.066693518Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=warn ts=2026-05-11T13:09:21.752319994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:23.319992276Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T13:09:23.320014299Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-11T13:09:26.070683152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:09:26.07864461Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-11T13:09:26.572513526Z caller=checkpoint.go:372 msg="writing series" size="339 kB"
level=warn ts=2026-05-11T13:09:26.752967956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:09:26.79208691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:09:26.796109404Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:09:28.366820927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:09:28.370990428Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-11T13:09:29.317842738Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:09:29.825138147Z caller=checkpoint.go:372 msg="writing series" size="282 kB"
level=info ts=2026-05-11T13:09:30.040434786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:09:30.04422115Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-11T13:09:31.752503784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:09:32.876088358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:09:32.880412063Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-05-11T13:09:33.077955436Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-05-11T13:09:36.33158786Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-11T13:09:36.75227432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:39.584526131Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=info ts=2026-05-11T13:09:40.867511009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:09:40.872709465Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:09:41.302024902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:09:41.75181002Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:09:41.751845675Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:09:41.751855651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:09:41.751880469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:09:42.382495022Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39268"
level=debug ts=2026-05-11T13:09:42.837637904Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-05-11T13:09:42.837661495Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-11T13:09:43.454482329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:09:43.458432788Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:09:43.63407849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:09:43.637996792Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:09:45.059240296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:09:45.062779947Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-11T13:09:45.38667207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T13:09:45.854532617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:09:45.862330164Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:09:46.090280912Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-11T13:09:46.752549456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:49.343967197Z caller=checkpoint.go:372 msg="writing series" size="500 kB"
level=info ts=2026-05-11T13:09:50.418108027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:09:50.428109762Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:09:51.752543959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:09:52.596897608Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-05-11T13:09:52.804863708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:09:52.808225537Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:09:55.849072598Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T13:09:56.752052114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:09:57.267177311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:09:57.272165031Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:09:59.102168415Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T13:09:59.319459651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T13:09:59.966048728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:09:59.976450044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:10:00.263692513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:10:00.267670296Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:10:00.934871145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:10:00.938430474Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-11T13:10:01.752334964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:02.187620494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:10:02.191668548Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:10:02.191693397Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:02.191719319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:10:02.195299192Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:10:02.355927584Z caller=checkpoint.go:372 msg="writing series" size="630 kB"
level=debug ts=2026-05-11T13:10:02.35595121Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-05-11T13:10:04.266547566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:10:04.270336666Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:10:04.562287094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:10:04.565686284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:10:05.162695608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:10:05.166129318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T13:10:05.609040765Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=info ts=2026-05-11T13:10:05.826808717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:10:05.830462634Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:10:06.331414956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:10:06.335437855Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:10:06.752360661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:08.01097144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:10:08.017951374Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T13:10:08.861048532Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-11T13:10:11.730036075Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:10:11.731167064Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:10:11.752031159Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:10:11.752067111Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-11T13:10:11.752063956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:11.752078616Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T13:10:12.114328469Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-11T13:10:12.114354577Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-11T13:10:12.23378881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:10:12.23870383Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:10:12.510631619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:10:15.063526279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:10:15.067481665Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-11T13:10:15.364814162Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51850"
level=debug ts=2026-05-11T13:10:15.370039217Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T13:10:15.370130884Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
ts=2026-05-11T13:10:15.482011155Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:47524"
level=info ts=2026-05-11T13:10:15.615900711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:10:15.620040015Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:10:16.203169866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:10:16.206697837Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T13:10:16.751949493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:10:18.620438133Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-11T13:10:21.752210973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:10:21.873546071Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T13:10:22.811876954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:10:22.820853979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:10:25.00847429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:10:25.013043247Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:10:25.145872514Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T13:10:25.145905229Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T13:10:25.318474219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:10:25.322422249Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:10:26.139698868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:10:26.144308298Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-05-11T13:10:26.144338461Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:26.144363307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:10:26.148327057Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:10:26.752235792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:27.180880926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:10:27.18509897Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:10:28.37964492Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-05-11T13:10:29.320962564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:10:31.632514567Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-11T13:10:31.752524844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:32.328405082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:10:32.329473221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:10:32.332935453Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:10:32.337859256Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T13:10:33.799055103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:10:33.808513688Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:10:34.885538994Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=debug ts=2026-05-11T13:10:36.499369441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-11T13:10:36.752639714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:10:38.138227596Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-11T13:10:39.683759737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:10:39.687715424Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:10:40.056071693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:10:40.063589188Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:10:40.063614521Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:40.063647637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:10:40.070764241Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-11T13:10:41.391897445Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
level=info ts=2026-05-11T13:10:41.752090052Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:10:41.752117652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:10:41.752125851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:10:41.752131592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:10:44.64501269Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-11T13:10:45.095325634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:10:45.100221231Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-05-11T13:10:45.483115761Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:49428"
level=warn ts=2026-05-11T13:10:46.752516055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:47.017736366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:10:47.021414152Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:10:47.482538821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:10:47.486647364Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:10:47.783701651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:10:47.786957801Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:10:47.897311815Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=debug ts=2026-05-11T13:10:48.169360081Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:10:48.169442135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="563 kB" avg_comp="563 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" idle=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:10:48.183093418Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=info ts=2026-05-11T13:10:50.325688271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:10:50.333055879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:10:50.525465949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:10:50.529218276Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:10:50.529241062Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:50.529268853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:10:50.532256145Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-11T13:10:51.151843192Z caller=checkpoint.go:372 msg="writing series" size="738 kB"
level=debug ts=2026-05-11T13:10:51.151867027Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-11T13:10:51.410694185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:10:51.415446349Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:10:51.469520173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:10:51.47555387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:10:51.475656251Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:10:51.478707668Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=warn ts=2026-05-11T13:10:51.752262551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:10:52.487880112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:10:52.491417234Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:10:52.491440835Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:52.491464159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:10:52.494423587Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:10:52.665047033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:10:52.668499175Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:10:53.178186968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:10:53.187680821Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:10:54.403458978Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-05-11T13:10:56.752329502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:10:57.656266366Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-11T13:10:57.896032842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:10:57.899730072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:10:57.899754471Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:57.899778494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:10:57.902447688Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
ts=2026-05-11T13:10:59.322704665Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:10:59.791935079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:10:59.79550049Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:10:59.795527557Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:10:59.795593985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:10:59.798719908Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:11:00.910088087Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T13:11:01.752952735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:11:04.163002978Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-11T13:11:04.877274732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:11:04.884239938Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=warn ts=2026-05-11T13:11:06.752640529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:11:07.415158264Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-11T13:11:07.492378064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:11:07.496560059Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:11:08.394537897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:11:08.398295617Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:11:09.076342247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:11:09.081025774Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:11:09.348911003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:11:09.351977575Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:11:09.577549982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:11:09.581315287Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:11:09.581343053Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:11:09.581377986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:11:09.584271969Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:11:10.145890043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:11:10.149282112Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:11:10.14930715Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:11:10.149330223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:11:10.152217585Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:11:10.50815743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:11:10.512476495Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:11:10.669125183Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-05-11T13:11:10.774618119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:11:10.778069259Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:11:11.456124976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:11:11.459508675Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-11T13:11:11.729420159Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:11:11.730476628Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:11:11.752679358Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T13:11:11.752692472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:11:11.752709637Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:11:11.752718428Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-11T13:11:12.600936285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:11:12.608993843Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:11:13.168247163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:11:13.175874336Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:11:13.921408597Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-11T13:11:14.322516979Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T13:11:15.484986266Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:59300"
level=warn ts=2026-05-11T13:11:16.752743741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:11:17.174882381Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-11T13:11:18.765716591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:11:18.771635785Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:11:20.415962464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:11:20.419727355Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-11T13:11:20.427824817Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-11T13:11:21.752738228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:21.76276794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:11:21.766281147Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:11:23.042532442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:11:23.04615431Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-11T13:11:23.235952209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:11:23.239782003Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:11:23.381192119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:11:23.389175255Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:11:23.680845074Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=debug ts=2026-05-11T13:11:23.680871627Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-11T13:11:26.752060589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:11:26.933162624Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-11T13:11:28.058832709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:11:28.066728512Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T13:11:29.323819237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:11:29.458336178Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:11:30.18793164Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T13:11:30.187954951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-11T13:11:31.752711344Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:32.55143695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:11:32.555984464Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=debug ts=2026-05-11T13:11:33.440020005Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T13:11:35.770589558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:11:35.773874829Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:11:35.773897176Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:11:35.773922765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:11:35.776636938Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:11:36.52198023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:11:36.52539057Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:11:36.692740136Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=warn ts=2026-05-11T13:11:36.752639975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:38.159850976Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:11:38.163380537Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:11:39.945443323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:11:39.946153306Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=error ts=2026-05-11T13:11:39.949956731Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:11:41.752334861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:11:41.752365621Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:11:41.752374002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:11:41.752370674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:42.181959546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:11:42.185509332Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:11:42.901166759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:11:42.906364427Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:11:43.138226242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:11:43.142316472Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T13:11:43.199120637Z caller=checkpoint.go:372 msg="writing series" size="875 kB"
level=debug ts=2026-05-11T13:11:43.199143535Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
ts=2026-05-11T13:11:45.369860954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58656"
level=debug ts=2026-05-11T13:11:46.358375893Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:11:46.455322864Z caller=checkpoint.go:372 msg="writing series" size="2.8 MB"
level=debug ts=2026-05-11T13:11:46.455385109Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=1
level=warn ts=2026-05-11T13:11:46.752535702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:46.999731753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:11:47.003651507Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:11:47.882163084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:11:47.886058006Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:11:48.045625687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:11:48.049696129Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:11:48.132004896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:11:48.1355455Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:11:49.472727769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:11:49.477303439Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:11:49.705290764Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=info ts=2026-05-11T13:11:50.962507346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:11:50.969529185Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:11:51.077758591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:11:51.081550677Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:11:51.682475866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:11:51.686695546Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:11:51.751895281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:11:52.232072867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:11:52.236007299Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:11:52.95800873Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=debug ts=2026-05-11T13:11:56.211468275Z caller=checkpoint.go:372 msg="writing series" size="576 kB"
level=info ts=2026-05-11T13:11:56.247453112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:11:56.251610772Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T13:11:56.752511135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:11:59.324993207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:11:59.464233179Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=info ts=2026-05-11T13:12:00.429002835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:12:00.437719179Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:12:01.751856174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:02.534771809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:12:02.539042313Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:12:02.539070155Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:02.539093416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:12:02.542531657Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:12:02.717013878Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-11T13:12:02.825533664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:12:02.83034927Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:12:05.975703493Z caller=checkpoint.go:372 msg="writing series" size="4.3 MB"
level=debug ts=2026-05-11T13:12:05.97576234Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.1 MB" series=6
level=warn ts=2026-05-11T13:12:06.752188807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:07.681514326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-11T13:12:07.68527367Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e16d525a3%3A19e16d52856%3Acbb0adb6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-11T13:12:07.685306629Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:07.68534214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:12:07.689248786Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-11T13:12:08.172017984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:12:08.180889794Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:12:08.180916696Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:08.180947466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:12:08.188724361Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:12:09.223479282Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-11T13:12:10.088483518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:12:10.097137541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:12:11.730214247Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:12:11.730212279Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:12:11.752280757Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:12:11.752311547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:12:11.752319856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:12:11.752364345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:12:12.475908605Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-11T13:12:15.729139241Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-11T13:12:16.504010603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:12:16.512120434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T13:12:16.751908295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:17.295014264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:12:17.298918766Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:12:17.530870921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:12:17.535899761Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:12:17.535935308Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:17.535962777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:12:17.539270149Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-11T13:12:18.205215015Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:12:18.652464391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:12:18.661265786Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:12:18.982653027Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-11T13:12:19.100621051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:12:19.104434945Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:12:19.104460425Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:19.104487145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:12:19.107756365Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T13:12:21.752595763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:22.024688506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:12:22.028321238Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:12:22.235534685Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-11T13:12:23.851749608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:12:23.855205081Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T13:12:25.059080797Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T13:12:25.196641621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:12:25.201387014Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-11T13:12:25.489006337Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=debug ts=2026-05-11T13:12:25.489029469Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-05-11T13:12:26.751995209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:27.520906064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:12:27.524834493Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:12:28.740716484Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-11T13:12:29.326698679Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:12:29.656355926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:12:29.659824792Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-11T13:12:29.687882317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:12:29.692134316Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=info ts=2026-05-11T13:12:30.266888597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:12:30.270401445Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:12:30.285428014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:12:30.289828991Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:12:30.930078254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:12:30.933473242Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:12:30.933499165Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:30.933526173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:12:30.936516472Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T13:12:31.752752133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:12:32.012871544Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T13:12:32.012900065Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-05-11T13:12:33.132528028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:12:33.136238247Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:12:35.24771771Z caller=checkpoint.go:372 msg="writing series" size="787 kB"
level=info ts=2026-05-11T13:12:35.79255489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:12:35.795972913Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:12:36.625625861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:12:36.628998072Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-11T13:12:36.752735316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:38.042867292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:12:38.048454057Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-11T13:12:38.499933695Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-05-11T13:12:39.079544293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:12:39.083988646Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:12:40.645487181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:12:40.649096747Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-11T13:12:41.75235514Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-11T13:12:41.752367476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:12:41.752382471Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:12:41.752391718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-11T13:12:41.752397022Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=2
level=info ts=2026-05-11T13:12:41.758398275Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-11T13:12:41.760035065Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007655982s
level=info ts=2026-05-11T13:12:42.333182558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:12:42.342386208Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:12:44.496875654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:12:44.501422712Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-11T13:12:45.003291729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:12:45.010477243Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-11T13:12:45.372309863Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59696"
level=info ts=2026-05-11T13:12:46.203083662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:12:46.20686095Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:12:46.579930086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:12:46.583726668Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:12:46.583747139Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:46.583768748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:12:46.586695242Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-11T13:12:46.752156633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:48.16587704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:12:48.170025228Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=info ts=2026-05-11T13:12:48.346546968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:12:48.351961885Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:12:51.75224648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:52.448399128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:12:52.453903345Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:12:52.453933656Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:12:52.453962396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:12:52.458045604Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:12:53.768524351Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:12:54.165625779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:12:54.169160273Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:12:55.077894788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:12:55.081892054Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-11T13:12:56.658381106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:12:56.66572768Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-11T13:12:56.75276952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:12:58.9506363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:12:58.95428659Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-11T13:12:59.328371508Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-60884486 10.128.0.89:7946"
level=info ts=2026-05-11T13:13:01.307857141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:13:01.315098899Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-11T13:13:01.751964696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:04.391464441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:13:04.395326767Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:13:06.651062236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:13:06.658812849Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-11T13:13:06.75293209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:07.243855917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:13:07.247187392Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:13:07.628775836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:13:07.632392408Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:13:07.720794717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:13:07.724384113Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:13:09.95604701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:13:09.961161074Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:13:10.053433209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-11T13:13:10.055515399Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-11T13:13:11.729666745Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:13:11.73072031Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:13:11.751600512Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-11T13:13:11.751773833Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-11T13:13:11.751784793Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-11T13:13:11.751803903Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.545µs success=true
level=info ts=2026-05-11T13:13:11.751864296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:13:11.751886792Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:13:11.751897225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:13:11.751930928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:13:11.752066702Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-11T13:13:15.005950051Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-05-11T13:13:16.752637847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:13:17.982628659Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:13:18.259030085Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=info ts=2026-05-11T13:13:20.511919413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:13:20.517212148Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:13:21.512069102Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-11T13:13:21.638740197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:13:21.642918325Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-11T13:13:21.7518689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:23.602315243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:13:23.60810576Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=info ts=2026-05-11T13:13:24.190884558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:13:24.195731057Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:13:24.195765827Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:13:24.195796301Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:13:24.199988794Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:13:24.765234053Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=info ts=2026-05-11T13:13:26.574501792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:13:26.579438354Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-11T13:13:26.752212592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:26.903542419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:13:26.911377404Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:13:27.548471844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:13:27.552647895Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:13:28.018246994Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-11T13:13:28.018270821Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-05-11T13:13:29.329288906Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:13:31.270469932Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=info ts=2026-05-11T13:13:31.687946462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:13:31.692838896Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:13:31.752225576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:32.030995763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:13:32.03506175Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:13:33.702459294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:13:33.706456534Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:13:34.197288589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:13:34.200835586Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:13:34.524127843Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-11T13:13:34.809037029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:13:34.813363914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:13:36.31114786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:13:36.315506949Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-11T13:13:36.752749034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:37.716594883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:13:37.723814641Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T13:13:37.777399059Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=info ts=2026-05-11T13:13:38.686806588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:13:38.69060809Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T13:13:41.029893287Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-11T13:13:41.752676106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:13:41.752707808Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:13:41.752716248Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:13:41.752779949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:41.833005733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:13:41.83731317Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-11T13:13:42.394310835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:39062"
level=info ts=2026-05-11T13:13:42.514220022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:13:42.518729436Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:13:43.170636226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:13:43.17905242Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:13:44.372373564Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-05-11T13:13:44.37241974Z caller=checkpoint.go:384 msg="flushing series" totalSize="69 MB" series=5
level=debug ts=2026-05-11T13:13:44.685198393Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-11T13:13:45.376147214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:51558"
level=info ts=2026-05-11T13:13:45.540922031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:13:45.552641318Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:13:45.854929061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:13:45.873318605Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:13:46.196721331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:13:46.209043487Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=warn ts=2026-05-11T13:13:46.752359704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:13:47.535629071Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-11T13:13:50.790483875Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-11T13:13:50.790538859Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=info ts=2026-05-11T13:13:50.815023629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:13:50.82622752Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T13:13:51.752518218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:13:52.086484073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:13:52.09434557Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:13:52.380277605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:13:52.399317377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:13:52.399368313Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:13:52.399433062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:13:52.418181078Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:13:53.86152476Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:13:54.074234247Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-11T13:13:54.074320294Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-05-11T13:13:54.115067263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:13:54.126968738Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=warn ts=2026-05-11T13:13:56.75227548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:13:57.295097679Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T13:13:57.536760792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:13:57.554671315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-11T13:13:59.332278719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-g82kl-bd40c277 10.128.0.86:7946"
level=debug ts=2026-05-11T13:14:00.547603049Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-11T13:14:01.751892337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:14:03.801088267Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-11T13:14:04.090120102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:14:04.10225226Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-11T13:14:06.752330858Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:14:07.054151401Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-11T13:14:08.466527584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:14:08.483628238Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:14:10.318329663Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-11T13:14:10.318374784Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=debug ts=2026-05-11T13:14:11.729462986Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:14:11.730517178Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:14:11.752747274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:14:11.752779165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:14:11.752787505Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:14:11.752925023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:14:12.395293231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.89:33776"
level=debug ts=2026-05-11T13:14:13.560055197Z caller=checkpoint.go:372 msg="writing series" size="525 kB"
level=info ts=2026-05-11T13:14:14.101831026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:14:14.105840584Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:14:14.627442518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:14:14.631133559Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:14:15.838991372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:14:15.846215083Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=warn ts=2026-05-11T13:14:16.752258576Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:14:16.812835086Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=info ts=2026-05-11T13:14:17.338451954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:14:17.342552939Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:14:18.997188375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:14:19.001561414Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-11T13:14:19.909276109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:14:19.913722551Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:14:19.913747978Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:14:19.913774231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:14:19.917952078Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:14:20.047452431Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:14:20.06566345Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=info ts=2026-05-11T13:14:20.310236758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:14:20.313673676Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:14:21.295549088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:14:21.303772178Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:14:21.303796136Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:14:21.303830966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:14:21.311845778Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-11T13:14:21.751966544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:14:22.727624782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:14:22.73307702Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-11T13:14:23.319353983Z caller=checkpoint.go:372 msg="writing series" size="713 kB"
level=debug ts=2026-05-11T13:14:23.31937894Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-11T13:14:24.40714105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:14:24.410608643Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-11T13:14:25.95568113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:14:25.959043201Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T13:14:26.572883959Z caller=checkpoint.go:372 msg="writing series" size="426 kB"
level=info ts=2026-05-11T13:14:26.663518297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:14:26.667406235Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=warn ts=2026-05-11T13:14:26.752005796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-11T13:14:29.342540903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:14:29.825059371Z caller=checkpoint.go:372 msg="writing series" size="315 kB"
level=info ts=2026-05-11T13:14:30.577797465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:14:30.581469106Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T13:14:31.752697164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:14:32.298526801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:14:32.303385689Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-11T13:14:32.43542138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:14:32.440060754Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:14:33.077919171Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=info ts=2026-05-11T13:14:33.520244053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="607 B" avg_comp="607 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:14:33.524155549Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:14:34.435864386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:14:34.443800383Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:14:35.76243029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:14:35.766533151Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:14:36.330729774Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-11T13:14:36.55744953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:14:36.560936361Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:14:36.697465428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:14:36.699602396Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-11T13:14:36.752625557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:14:37.181227047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:14:37.184927855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:14:37.188633931Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:14:37.192689253Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:14:37.659678445Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:14:39.584288324Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
level=debug ts=2026-05-11T13:14:39.584311732Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-05-11T13:14:41.7521572Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:14:41.752188389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:14:41.752196865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:14:41.752226874Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:14:42.430496239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:14:42.439248878Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-11T13:14:42.836659138Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=info ts=2026-05-11T13:14:44.032154987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:14:44.036229725Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:14:46.090528783Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=warn ts=2026-05-11T13:14:46.752661583Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:14:47.94690876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:14:47.956976616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:14:48.638928147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:14:48.643314623Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:14:49.342989037Z caller=checkpoint.go:372 msg="writing series" size="596 kB"
level=warn ts=2026-05-11T13:14:51.752989668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:14:52.414619277Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:14:52.59629012Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=debug ts=2026-05-11T13:14:52.596313206Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-11T13:14:52.64747208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:14:52.651512608Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:14:52.651550217Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:14:52.651598323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:14:52.65472681Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=debug ts=2026-05-11T13:14:55.849295617Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-11T13:14:56.75232195Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:14:59.10206215Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-11T13:14:59.344491637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T13:15:00.977268315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:15:00.98313004Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-11T13:15:01.752981554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:02.355606767Z caller=checkpoint.go:372 msg="writing series" size="647 kB"
level=info ts=2026-05-11T13:15:02.711994432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:15:02.716187311Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-11T13:15:03.246797302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:15:03.252054294Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=info ts=2026-05-11T13:15:03.815791841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:15:03.819974964Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:15:04.189878961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:15:04.194065729Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-11T13:15:05.608794154Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-05-11T13:15:05.608818661Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=info ts=2026-05-11T13:15:06.030375905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:15:06.034184632Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:15:06.303542648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:15:06.307662663Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-05-11T13:15:06.752211336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:08.861116279Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-05-11T13:15:09.790011542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:15:09.793578033Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-11T13:15:11.729736413Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:15:11.7307733Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-11T13:15:11.732077382Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-11T13:15:11.732121299Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976116 n=1
level=debug ts=2026-05-11T13:15:11.732163839Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976116 n=1
level=info ts=2026-05-11T13:15:11.752822836Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:15:11.752850304Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:15:11.75285787Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:15:11.75290395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:12.114244294Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-11T13:15:12.538250897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:15:12.544132904Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-11T13:15:15.368021852Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-11T13:15:15.368046294Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=3
ts=2026-05-11T13:15:15.381229996Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44348"
level=info ts=2026-05-11T13:15:15.989293913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.2 MB" avg_comp="2.1 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:15:15.99741982Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:15:16.101089895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:15:16.104629815Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-11T13:15:16.75240372Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:15:17.119955427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:15:17.12424805Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:15:18.083496004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:15:18.088621535Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-11T13:15:18.620359525Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-11T13:15:18.777998345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:15:18.787372494Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-11T13:15:19.367289308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-11T13:15:20.226373292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:15:20.231354864Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-11T13:15:20.679535081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:15:20.684200935Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T13:15:21.752709367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:21.873013354Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-11T13:15:23.180040527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:15:23.183944828Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-11T13:15:23.316229538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:15:23.319713867Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=debug ts=2026-05-11T13:15:25.146596329Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-11T13:15:25.146626785Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=3
level=info ts=2026-05-11T13:15:25.778769167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:15:25.782182202Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-11T13:15:26.416180768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:15:26.419612217Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-11T13:15:26.752218456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:15:28.211004512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:15:28.216887689Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:15:28.216918699Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-11T13:15:28.21695529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:15:28.223822869Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-11T13:15:28.379176732Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-11T13:15:29.193874084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:15:29.201878099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T13:15:29.345855065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:15:30.31073091Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-11T13:15:31.632418468Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=warn ts=2026-05-11T13:15:31.752155604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:34.88571442Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-05-11T13:15:36.752496838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:15:38.098179697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:15:38.102864226Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"
level=debug ts=2026-05-11T13:15:38.13828612Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-05-11T13:15:39.255480382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ef6e100902f59363 immediate=false num_chunks=1 total_comp="20 kB" avg_comp="20 kB" total_uncomp="67 kB" avg_uncomp="67 kB" full=1 labels="{stream=\"build-e2e-flvz\"}"
level=error ts=2026-05-11T13:15:39.258945668Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ef6e100902f59363/19e16de5bd0%3A19e16e95b1a%3Af5b668b1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-flvz\"}"
level=info ts=2026-05-11T13:15:40.176658736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:15:40.184279828Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-11T13:15:41.066465722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:15:41.069970152Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:15:41.390894132Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-05-11T13:15:41.752675581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:15:41.752711181Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:15:41.75271993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:15:41.752731028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:43.676300075Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:15:44.644660506Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-11T13:15:45.382241383Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57712"
level=warn ts=2026-05-11T13:15:46.752168651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:15:47.161355324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="137 MB" avg_uncomp="23 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-11T13:15:47.169875911Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e16d4a024%3A19e16d51f36%3Aa3f333d5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-11T13:15:47.50423715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="386 kB" avg_comp="386 kB" total_uncomp="2.0 MB" avg_uncomp="2.0 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-05-11T13:15:47.508235539Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19e16d4a098%3A19e1706c6cd%3A8b536879?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-05-11T13:15:47.784305145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:15:47.788817265Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=debug ts=2026-05-11T13:15:47.897118146Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-11T13:15:50.995191425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-11T13:15:51.000229688Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e16d52566%3A19e16d54a62%3A775c6b33?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-11T13:15:51.151308581Z caller=checkpoint.go:372 msg="writing series" size="738 kB"
level=debug ts=2026-05-11T13:15:51.151341666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-11T13:15:51.752821303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:15:52.809633646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="466 kB" avg_comp="466 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-11T13:15:52.814419237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e16d48eb7%3A19e1727fb79%3A3c8efd41?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-11T13:15:54.403444848Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-11T13:15:56.752321579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:15:57.656517176Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-11T13:15:57.721768964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="9.8 kB" avg_comp="9.8 kB" total_uncomp="65 kB" avg_uncomp="65 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-11T13:15:57.726338808Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e16d52b40%3A19e16ebec78%3Afd822fbf?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-11T13:15:58.067881537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.2 MB" avg_comp="2.1 MB" total_uncomp="28 MB" avg_uncomp="14 MB" full=1 synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-11T13:15:58.075885944Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e16d4941c%3A19e16f2d2d0%3A9a55bbf3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-11T13:15:58.853725341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="28 kB" avg_uncomp="28 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-11T13:15:58.857111152Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e16d49d90%3A19e16d522a1%3Accb85bae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-11T13:15:59.291339591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="459 kB" avg_comp="459 kB" total_uncomp="4.7 MB" avg_uncomp="4.7 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-11T13:15:59.295580691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e16d49d67%3A19e16f5596b%3A83046616?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-11T13:15:59.347230744Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=info ts=2026-05-11T13:15:59.372957482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="608 kB" avg_comp="608 kB" total_uncomp="4.0 MB" avg_uncomp="4.0 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-11T13:15:59.377173513Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e16d4909a%3A19e172128b4%3A8302cc7a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-11T13:16:00.909629294Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-11T13:16:01.752489987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:16:04.162214865Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-11T13:16:06.218114162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=2 total_comp="844 kB" avg_comp="422 kB" total_uncomp="4.4 MB" avg_uncomp="2.2 MB" synced=2 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-11T13:16:06.222671963Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e16d492f9%3A19e16eac27a%3A8b6c1f10?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-11T13:16:06.752035779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:16:07.415439874Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-11T13:16:08.695212502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-11T13:16:08.699951729Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e16d52566%3A19e16d52b42%3A50b1b43?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-11T13:16:09.19762763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="625 B" avg_comp="625 B" total_uncomp="1.6 kB" avg_uncomp="1.6 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-11T13:16:09.201152674Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e16d52273%3A19e16d52274%3Ac38ddd0d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-11T13:16:10.099593252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="29 kB" avg_comp="29 kB" total_uncomp="161 kB" avg_uncomp="161 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-11T13:16:10.103161072Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e16d51753%3A19e16d52567%3Af9678018?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-11T13:16:10.326123004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-11T13:16:10.66807118Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-11T13:16:11.729485757Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.70:11211 count=1
level=info ts=2026-05-11T13:16:11.730589635Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-11T13:16:11.752429019Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-11T13:16:11.752465509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-11T13:16:11.752475115Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-11T13:16:11.752497152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:16:13.921004208Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-11T13:16:14.613617992Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-11T13:16:15.346016519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-11T13:16:15.349303894Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e16dde69f%3A19e16df3691%3A650db170?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-05-11T13:16:15.500078774Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.87:35160"
level=info ts=2026-05-11T13:16:15.512902735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=004355b7f789f993 immediate=false num_chunks=1 total_comp="5.4 kB" avg_comp="5.4 kB" total_uncomp="12 kB" avg_uncomp="12 kB" full=1 labels="{stream=\"build-e2e-ychv\"}"
level=error ts=2026-05-11T13:16:15.516389466Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/4355b7f789f993/19e16de9280%3A19e16e07e05%3Aeb431236?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-ychv\"}"
level=info ts=2026-05-11T13:16:15.840549519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="76 MB" avg_uncomp="19 MB" full=4 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-11T13:16:15.848104554Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e16d4999c%3A19e1701bedb%3Ab49f870a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-11T13:16:15.86914985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="2.0 MB" avg_comp="2.0 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-11T13:16:15.876178306Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e16d48f85%3A19e170e7451%3Ac47394fd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-11T13:16:16.752015589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:16:16.824398411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-11T13:16:16.827927807Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e16d525a3%3A19e16d52b42%3Aa5d0cc37?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-11T13:16:17.174336125Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-11T13:16:17.624917829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="383 kB" avg_comp="383 kB" total_uncomp="3.5 MB" avg_uncomp="3.5 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-11T13:16:17.630355397Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e16d49dc3%3A19e16f99bb5%3A6d6a5ad7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-11T13:16:20.427676102Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-11T13:16:20.573129329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-11T13:16:20.576831943Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e16d52273%3A19e16e7e150%3A5b24e72c?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-11T13:16:21.752601427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-11T13:16:22.6907512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-11T13:16:22.695581206Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e16d51753%3A19e16d52567%3Aa3bfc9ff?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-11T13:16:23.682299504Z caller=checkpoint.go:372 msg="writing series" size="592 kB"
level=debug ts=2026-05-11T13:16:23.682331464Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-05-11T13:16:26.752530138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:16:26.933621138Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-11T13:16:28.849634352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7de7c9aa96d88818 immediate=false num_chunks=1 total_comp="592 kB" avg_comp="592 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-lpvy\"}"
level=error ts=2026-05-11T13:16:28.854615119Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7de7c9aa96d88818/19e16dff210%3A19e16f1e69b%3Ab8b7c3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-lpvy\"}"
level=info ts=2026-05-11T13:16:28.878656947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=25 total_comp="68 MB" avg_comp="2.7 MB" total_uncomp="562 MB" avg_uncomp="22 MB" full=23 synced=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-11T13:16:28.886171049Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e16d48eb5%3A19e16de0177%3Aae712d09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 25, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-11T13:16:29.348986141Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-dk5vr-8ae7ce43 10.128.0.87:7946"
level=debug ts=2026-05-11T13:16:30.187662792Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-11T13:16:30.187687654Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-11T13:16:31.75196168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-11T13:16:33.439446773Z caller=checkpoint.go:372 msg="writing series" size="939 B"
level=info ts=2026-05-11T13:16:33.58296047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=905361dd80838f9d immediate=false num_chunks=1 total_comp="202 kB" avg_comp="202 kB" total_uncomp="908 kB" avg_uncomp="908 kB" full=1 labels="{stream=\"integration1-cagf\"}"
level=error ts=2026-05-11T13:16:33.589124766Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/905361dd80838f9d/19e16ddf258%3A19e16e786cd%3A18e84134?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-cagf\"}"