level=info ts=2026-05-10T18:42:49.108916416Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-05-10T18:42:49.108942069Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-05-10T18:42:49.109251623Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-05-10T18:42:49.110130884Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-05-10T18:42:49.11050019Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-10T18:42:49.110575506Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0
level=info ts=2026-05-10T18:42:49.111107113Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-c8159ab3
ts=2026-05-10T18:42:49.111178942Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-05-10T18:42:49.111190451Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946
level=debug ts=2026-05-10T18:42:49.111210535Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.75 advertisePort=7946
level=info ts=2026-05-10T18:42:49.111324991Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-05-10T18:42:49.115717421Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T18:42:49.115742104Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-05-10T18:42:49.116928706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T18:42:49.116901107Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-05-10T18:42:49.117663248Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-10T18:42:49.117679794Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-10T18:42:49.117678287Z
level=debug ts=2026-05-10T18:42:49.117688484Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-05-10T18:42:49.117703967Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-05-10T18:42:49.117716846Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-05-10T18:42:49.117721385Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-05-10T18:42:49.117720545Z
level=debug ts=2026-05-10T18:42:49.117725229Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-05-10T18:42:49.118704095Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T18:42:49.118724784Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:42:49.118734948Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.412208ms
level=error ts=2026-05-10T18:42:49.118744704Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-05-10T18:42:49.11875556Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-05-10T18:42:49.118926875Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-05-10T18:42:49.118995823Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.75 inf=eth0
ts=2026-05-10T18:42:49.120900966Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:42:49.12091483Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-10T18:42:49.132346368Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-05-10T18:42:49.132392906Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-05-10T18:42:49.132400576Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-05-10T18:42:49.132430532Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-05-10T18:42:49.1324542Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-05-10T18:42:49.132460988Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-05-10T18:42:49.132474566Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-05-10T18:42:49.132686368Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-05-10T18:42:49.132695928Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-05-10T18:42:49.132598168Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-05-10T18:42:49.132710452Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-05-10T18:42:49.132712613Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-05-10T18:42:49.132716428Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-05-10T18:42:49.132719334Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-05-10T18:42:49.132784196Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-05-10T18:42:49.132852979Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-05-10T18:42:49.132870108Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-05-10T18:42:49.132873927Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-05-10T18:42:49.132876866Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-05-10T18:42:49.132879673Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-05-10T18:42:49.132961346Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-05-10T18:42:49.132965512Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-05-10T18:42:49.13296833Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-05-10T18:42:49.132971085Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-05-10T18:42:49.132973684Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-05-10T18:42:49.132982555Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-05-10T18:42:49.132991286Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T18:42:49.133033919Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.204µs success=true
level=info ts=2026-05-10T18:42:49.1330472Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-05-10T18:42:49.133089274Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=107.83µs errors=false
level=info ts=2026-05-10T18:42:49.133094475Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-05-10T18:42:49.133171278Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=189.869µs errors=false
level=info ts=2026-05-10T18:42:49.133177512Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-05-10T18:42:49.133182472Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=200.839µs
level=info ts=2026-05-10T18:42:49.133202085Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-05-10T18:42:49.133217652Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-05-10T18:42:49.133224837Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=23.391299479s
level=info ts=2026-05-10T18:42:49.13323479Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-05-10T18:42:49.133312638Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-05-10T18:42:49.13331431Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-05-10T18:42:49.133318465Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-05-10T18:42:49.133324652Z caller=loki.go:615 msg="Loki started" startup_time=82.99013ms
level=debug ts=2026-05-10T18:42:49.133393378Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-05-10T18:42:50.694184816Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:42:50.694211412Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-05-10T18:42:53.604852231Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:42:53.604894726Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:42:54.133712369Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:42:58.163081364Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:42:58.163105266Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:42:59.134346703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:04.133664766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:09.13430932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:43:11.432600188Z caller=logging.go:137 msg="GET /ready (503) 109.677µs"
ts=2026-05-10T18:43:13.754981277Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.76:7946"
ts=2026-05-10T18:43:13.757724829Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.78:7946: dial tcp 10.130.0.78:7946: connect: connection refused"
ts=2026-05-10T18:43:13.759621329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.66:7946"
ts=2026-05-10T18:43:13.762456932Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.67:7946: dial tcp 10.128.0.67:7946: connect: connection refused"
ts=2026-05-10T18:43:13.76370633Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.128.0.70:7946: dial tcp 10.128.0.70:7946: connect: connection refused"
ts=2026-05-10T18:43:13.76491604Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.69:7946"
level=info ts=2026-05-10T18:43:13.766064026Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=24.647303873s
level=warn ts=2026-05-10T18:43:14.133807631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:43:19.133347335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-05-10T18:43:19.133381122Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-05-10T18:43:19.133390771Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:43:19.133391925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:43:21.430465176Z caller=logging.go:137 msg="GET /ready (503) 74.021µs"
level=warn ts=2026-05-10T18:43:24.133376012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:29.133542759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:34.134285267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:39.133410631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:44.133674241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:43:44.562944913Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T18:43:44.562986782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:43:44.569556198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:43:45.565767934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-10T18:43:48.806933407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:43:49.117177058Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:43:49.11761751Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:43:49.133954543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:43:49.133976889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:43:49.133985556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:43:49.133978794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:54.1343157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:43:59.133450884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:44:02.807578097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:44:02.817138622Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:44:04.133812839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:44:04.720347724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:44:09.133415291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:44:14.13402828Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:44:18.810045939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:44:19.134216334Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:44:19.134243061Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:44:19.134252601Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:44:19.134473631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:44:19.890282188Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:44:23.973218698Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:44:24.134031456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:44:29.133491629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:44:33.407364835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:44:33.41407576Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:44:34.133700467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:44:39.133851697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:44:44.134080497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:44:46.919356458Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-05-10T18:44:48.812565607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:44:49.117067051Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:44:49.117877413Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:44:49.134168083Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:44:49.13416709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:44:49.134190945Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:44:49.13419887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:44:52.86452911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:44:54.133545295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:44:59.13340096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:04.133823368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:09.134149189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:14.133808596Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:45:14.847107873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37032"
level=debug ts=2026-05-10T18:45:16.984800657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T18:45:17.550751618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:45:17.55896771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T18:45:18.815133209Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:45:19.13404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:45:19.134050003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:45:19.134068929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:45:19.134077164Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:45:21.212175535Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-10T18:45:21.213743969Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:45:24.134400533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:29.134148278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:34.133827589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:39.133395317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:45:43.486484583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:45:44.134370298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:45:47.621340278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47332"
ts=2026-05-10T18:45:48.817775859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:45:49.116992917Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:45:49.11725059Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T18:45:49.118483831Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T18:45:49.118536115Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976042 n=1
level=debug ts=2026-05-10T18:45:49.118575831Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976042 n=1
level=info ts=2026-05-10T18:45:49.133526245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:45:49.133543587Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T18:45:49.133537234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:45:49.133550987Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:45:54.133996617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:45:59.133829523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:45:59.639900776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:46:04.134152368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:46:09.133704182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:46:12.086536363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:46:12.095403955Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:46:13.652947718Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:46:14.133740511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:46:18.821022546Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:46:19.133362283Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:46:19.133370784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:46:19.133388026Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:46:19.133396373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:46:24.133413658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:46:29.134335422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:46:34.133748831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:46:38.773580196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:46:39.134090237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:46:44.134211414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:46:48.824126501Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T18:46:49.116933762Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:46:49.116965366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T18:46:49.134273292Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:46:49.134285488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:46:49.13429207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:46:49.134291989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:46:54.134036228Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:46:54.884259653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:46:54.892777355Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:46:55.858218055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:46:59.133533397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:47:03.80593595Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:47:04.134029194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:47:09.133586802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:47:14.13339553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:47:14.855446836Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57902"
ts=2026-05-10T18:47:18.82700101Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=warn ts=2026-05-10T18:47:19.133558755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:47:19.133559519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:47:19.133585906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:47:19.133596698Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:47:24.13341031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:47:29.133728841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:47:32.530872398Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:47:34.133423094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:47:39.134248033Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:47:44.13379641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:47:47.629987452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53212"
ts=2026-05-10T18:47:48.830217225Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:47:49.11702637Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:47:49.117758355Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:47:49.134054736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:47:49.134068758Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:47:49.134076474Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T18:47:49.134089121Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T18:47:49.134087849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:47:49.134239416Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-10T18:47:49.134248046Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T18:47:49.134262351Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=9.516µs success=true
level=debug ts=2026-05-10T18:47:49.134447353Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=info ts=2026-05-10T18:47:49.248435917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:47:49.256071274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:47:53.489495907Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T18:47:54.134152039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:47:57.844713133Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=warn ts=2026-05-10T18:47:59.134363071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:47:59.149911527Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-05-10T18:48:02.199597428Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
ts=2026-05-10T18:48:03.379102307Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58716"
level=debug ts=2026-05-10T18:48:03.836790542Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:48:04.133675877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:06.554822285Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-10T18:48:09.133717402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:10.909727675Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-10T18:48:14.133705923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:15.264239829Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T18:48:18.833212571Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:48:19.133354554Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:48:19.133383998Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:48:19.133395335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:48:19.133412789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:19.619417789Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-10T18:48:23.973857688Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T18:48:24.13357623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:28.328588351Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-05-10T18:48:29.133364658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:32.683256413Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=debug ts=2026-05-10T18:48:33.085280618Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T18:48:33.940812598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:48:33.953995363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:48:34.134272242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:37.038086007Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-10T18:48:39.133682637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:41.392837524Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-10T18:48:44.134304727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:45.74808409Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-10T18:48:45.748108304Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
ts=2026-05-10T18:48:48.836456527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:48:49.117415341Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:48:49.118017063Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:48:49.134280203Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:48:49.134293025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:49.134304538Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:48:49.134312981Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:48:50.103254115Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T18:48:54.134309211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:48:54.457817686Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-10T18:48:58.812814553Z caller=checkpoint.go:372 msg="writing series" size="262 kB"
level=warn ts=2026-05-10T18:48:59.133589084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:03.167117443Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
ts=2026-05-10T18:49:03.383593807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59456"
level=warn ts=2026-05-10T18:49:04.13383766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:06.162148781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:49:07.521772559Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=warn ts=2026-05-10T18:49:09.13421067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:11.876889092Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=debug ts=2026-05-10T18:49:12.192168561Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-05-10T18:49:14.134027604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:49:14.862461421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55494"
level=debug ts=2026-05-10T18:49:16.231902262Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-05-10T18:49:18.839083075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:49:19.133738386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:49:19.133755875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:19.133769326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:49:19.13377908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:49:20.589970751Z caller=checkpoint.go:372 msg="writing series" size="5.4 MB"
level=debug ts=2026-05-10T18:49:20.589992763Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.0 MB" series=8
level=warn ts=2026-05-10T18:49:24.134200944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:24.943200553Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-10T18:49:24.943237517Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=1
level=warn ts=2026-05-10T18:49:29.13362306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:29.296033636Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=info ts=2026-05-10T18:49:29.524033492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:49:29.533126801Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:49:33.651577401Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-10T18:49:34.134222192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:38.00553681Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-05-10T18:49:38.347603232Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:49:39.133812246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:42.360651799Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-05-10T18:49:44.134174961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:46.715549354Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
ts=2026-05-10T18:49:48.841481523Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:49:49.117657531Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:49:49.118420769Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:49:49.133626778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:49:49.133656579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:49.133669632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:49:49.133680116Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:49:51.070585948Z caller=checkpoint.go:372 msg="writing series" size="206 kB"
level=warn ts=2026-05-10T18:49:54.13406743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:55.425705972Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T18:49:59.133405091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:49:59.7804771Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
ts=2026-05-10T18:50:03.387958065Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42226"
level=warn ts=2026-05-10T18:50:04.133951568Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:04.134977008Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-10T18:50:08.490268864Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T18:50:09.134155423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:11.991501479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:50:12.844774268Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=debug ts=2026-05-10T18:50:13.046543598Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-05-10T18:50:13.048229123Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:50:14.133597793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:17.199512462Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
ts=2026-05-10T18:50:17.638617808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40632"
ts=2026-05-10T18:50:18.844261331Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:50:19.134206125Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:50:19.134236177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:50:19.134243934Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:50:19.134236793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:50:20.696794117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:50:20.70411618Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:50:20.704142701Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T18:50:20.704164088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:50:20.711173458Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:50:21.554073821Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T18:50:24.134193725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:25.909832549Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=debug ts=2026-05-10T18:50:25.909854512Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-10T18:50:29.133383691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:30.264121346Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
level=warn ts=2026-05-10T18:50:34.133761358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:50:34.330550235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:50:34.341587894Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:50:34.618357703Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T18:50:38.973222001Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T18:50:39.133479099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:42.763596988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:50:43.328807552Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-10T18:50:44.134246357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:47.683104703Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
ts=2026-05-10T18:50:48.847511178Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:50:49.117172577Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:50:49.117923153Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:50:49.134201509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:50:49.134212589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:49.134218902Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:50:49.13422685Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:50:52.038150377Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T18:50:54.134005801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:50:56.393085199Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T18:50:59.133402255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:00.7479065Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=debug ts=2026-05-10T18:51:02.136238597Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T18:51:03.392545608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58570"
level=warn ts=2026-05-10T18:51:04.133625184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:05.102191268Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T18:51:09.133945309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:09.457787632Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-10T18:51:10.46789693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:51:10.477253643Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:51:11.169533763Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:51:13.812070618Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T18:51:14.13400356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:51:14.870335911Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36450"
level=debug ts=2026-05-10T18:51:18.167411545Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
ts=2026-05-10T18:51:18.850099494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:51:19.133600306Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:51:19.133626495Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:51:19.133649322Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:51:19.133623251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:22.521811721Z caller=checkpoint.go:372 msg="writing series" size="129 kB"
level=warn ts=2026-05-10T18:51:24.134036601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:26.877390978Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-05-10T18:51:29.134126684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:31.232205543Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
ts=2026-05-10T18:51:33.395919111Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39770"
level=warn ts=2026-05-10T18:51:34.133824785Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:35.586700925Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=debug ts=2026-05-10T18:51:37.846221782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:51:39.134402938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:39.94111928Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T18:51:39.941140968Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=17
level=warn ts=2026-05-10T18:51:44.13422309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:44.296625282Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
ts=2026-05-10T18:51:44.873375592Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44462"
level=debug ts=2026-05-10T18:51:48.65070864Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-10T18:51:48.853326268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:51:49.117378186Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:51:49.117882911Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:51:49.134164062Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:51:49.134181025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:51:49.134188704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:51:49.134248971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:51:53.005505644Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-10T18:51:54.134169054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:51:56.409759651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:51:56.418685361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:51:57.360581067Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T18:51:59.133956692Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:00.377056433Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T18:52:01.715808393Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=debug ts=2026-05-10T18:52:02.887373551Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:52:04.134373378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:06.070804611Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-05-10T18:52:09.13410143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:10.425485541Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-10T18:52:14.133999135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:14.780112262Z caller=checkpoint.go:372 msg="writing series" size="420 kB"
ts=2026-05-10T18:52:18.856443561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:52:19.134105295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:52:19.134120915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:19.134141351Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:52:19.134150273Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:52:19.135210745Z caller=checkpoint.go:384 msg="flushing series" totalSize="836 kB" series=8
level=info ts=2026-05-10T18:52:19.140784706Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-05-10T18:52:19.143925121Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009536717s
level=warn ts=2026-05-10T18:52:24.1343376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:52:29.134339746Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:52:30.16701377Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:52:34.134309581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:52:39.133535006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:52:44.133586017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:52:44.878527697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53776"
ts=2026-05-10T18:52:47.647723256Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44338"
ts=2026-05-10T18:52:48.859056386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:52:49.117840032Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:52:49.118337935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:52:49.133586027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:52:49.133605267Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:52:49.133612948Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T18:52:49.133623559Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T18:52:49.133629796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:52:49.13376441Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-10T18:52:49.133773336Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T18:52:49.133791566Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.409µs success=true
level=debug ts=2026-05-10T18:52:49.133953153Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-10T18:52:53.489131428Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T18:52:54.134332774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:52:55.881241191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:52:55.892585954Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:52:57.843942167Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T18:52:58.994537347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T18:52:59.134113836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:02.198625485Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-05-10T18:53:04.133437208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:04.209722567Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:53:06.55372926Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-10T18:53:09.134070699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:10.908317813Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-10T18:53:14.13350962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:15.263226191Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T18:53:17.650760915Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40212"
ts=2026-05-10T18:53:18.861847433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:53:19.134143246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:53:19.134157448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:19.134175367Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:53:19.134185058Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:53:19.618585543Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-10T18:53:23.972851421Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-10T18:53:24.134363185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:28.328547087Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=debug ts=2026-05-10T18:53:29.102281004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:53:29.133525851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:32.683362025Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-10T18:53:32.683389907Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-05-10T18:53:34.133755706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:37.037383259Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=warn ts=2026-05-10T18:53:39.133369607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:53:40.892601042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:53:40.904103402Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:53:41.392808655Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-10T18:53:44.133828329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:44.203974822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T18:53:45.746906471Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-05-10T18:53:48.864259979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:53:49.117013069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:53:49.117844534Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:53:49.134194726Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:53:49.134216761Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:53:49.134225172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:53:49.134221536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:50.102711329Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T18:53:54.133477917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:53:54.456591207Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-10T18:53:58.812357739Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=warn ts=2026-05-10T18:53:59.13440229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:03.167265534Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
ts=2026-05-10T18:54:03.406995739Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50300"
level=warn ts=2026-05-10T18:54:04.133496954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:04.782154687Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:54:07.522062291Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-10T18:54:09.133735504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:11.876125529Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-10T18:54:14.134262292Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:16.231798977Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
ts=2026-05-10T18:54:18.866823564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:54:19.134114439Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:54:19.134145217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:54:19.134152501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:54:19.134152003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:20.590517489Z caller=checkpoint.go:372 msg="writing series" size="6.4 MB"
level=debug ts=2026-05-10T18:54:20.590541079Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.5 MB" series=11
level=warn ts=2026-05-10T18:54:24.13359137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:24.941655462Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-10T18:54:24.941678237Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=warn ts=2026-05-10T18:54:29.133505663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:29.295891915Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=debug ts=2026-05-10T18:54:33.650961753Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-10T18:54:34.134217373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:54:37.731387874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:54:37.742922882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:54:38.005597398Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=warn ts=2026-05-10T18:54:39.133382185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:40.404505557Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T18:54:40.406105611Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T18:54:40.407168817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:54:42.360079325Z caller=checkpoint.go:372 msg="writing series" size="270 kB"
level=warn ts=2026-05-10T18:54:44.134329415Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:54:44.885355847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56476"
level=debug ts=2026-05-10T18:54:46.715305016Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
ts=2026-05-10T18:54:47.656327042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53394"
ts=2026-05-10T18:54:48.869223713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:54:49.1170314Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:54:49.117462618Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:54:49.133945361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:54:49.133964379Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:54:49.13397411Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:54:49.133976389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:51.070076267Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-10T18:54:54.13395603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:55.425190487Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T18:54:59.133415792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:54:59.77986197Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-05-10T18:55:04.134146057Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T18:55:04.134182595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:06.585444808Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:55:08.489215693Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T18:55:09.13413241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:12.844086819Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-10T18:55:14.13421955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:17.199499673Z caller=checkpoint.go:372 msg="writing series" size="113 kB"
ts=2026-05-10T18:55:18.872133608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:55:19.133385773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:55:19.133387078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:19.133412109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:55:19.133420224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:55:21.554155478Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T18:55:24.133983772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:55:25.476053485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:55:25.48652609Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:55:25.908792973Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=debug ts=2026-05-10T18:55:25.90881443Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=14
level=warn ts=2026-05-10T18:55:29.134305744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:30.264055431Z caller=checkpoint.go:372 msg="writing series" size="8.3 kB"
level=debug ts=2026-05-10T18:55:33.365774133Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T18:55:33.412968658Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35554"
level=warn ts=2026-05-10T18:55:34.133486207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:34.618843349Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T18:55:38.972936038Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T18:55:39.133729801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:40.326789004Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:55:43.327829962Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-10T18:55:44.133700963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:47.683083431Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
ts=2026-05-10T18:55:48.874500881Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:55:49.117089285Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:55:49.117715308Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:55:49.13418428Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:55:49.134205546Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:55:49.134215266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:55:49.134251253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:52.037343183Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T18:55:54.134284845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:55:56.392541184Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T18:55:59.133457034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:00.747647311Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=warn ts=2026-05-10T18:56:04.133772489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:05.102690116Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T18:56:09.060040809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:56:09.070660825Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:56:09.133816441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:09.457507953Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-10T18:56:13.811581149Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T18:56:14.133815962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:15.004383917Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T18:56:17.662391277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:58972"
level=debug ts=2026-05-10T18:56:18.166425013Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-05-10T18:56:18.877137615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:56:19.133361365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:56:19.13337272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:19.133386158Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:56:19.133393974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:56:21.68904046Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T18:56:22.522122916Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-05-10T18:56:24.133529353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:26.876425258Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-10T18:56:29.134075165Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:31.231265063Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
ts=2026-05-10T18:56:33.417137347Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33244"
level=warn ts=2026-05-10T18:56:34.133825386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:35.585828488Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-05-10T18:56:39.134266179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:39.940980043Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-05-10T18:56:39.941004422Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=17
level=warn ts=2026-05-10T18:56:44.134367491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:44.295821091Z caller=checkpoint.go:372 msg="writing series" size="230 kB"
level=debug ts=2026-05-10T18:56:46.17298018Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T18:56:47.665350053Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43010"
level=debug ts=2026-05-10T18:56:48.650578528Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-05-10T18:56:48.880163611Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:56:49.117379944Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:56:49.11839409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T18:56:49.133735303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:56:49.133742101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:56:49.133768125Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:56:49.133777895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:56:53.005577802Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-10T18:56:54.134279218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:57.360080056Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T18:56:57.62295929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:56:57.631724433Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:56:57.631751012Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T18:56:57.631779858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:56:57.639163607Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:56:59.134047656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:56:59.917448723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T18:57:01.714797926Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-10T18:57:04.133731495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:57:06.069715952Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-10T18:57:09.13349172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:57:10.288220908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:57:10.297435916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:57:10.425040948Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-05-10T18:57:14.133453138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:57:14.780079618Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
ts=2026-05-10T18:57:18.882523972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T18:57:18.920673579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T18:57:19.133608869Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:57:19.133644365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:57:19.133654496Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:57:19.133666953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:57:19.13476322Z caller=checkpoint.go:384 msg="flushing series" totalSize="959 kB" series=8
level=info ts=2026-05-10T18:57:19.140964256Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-05-10T18:57:19.142367222Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008463519s
level=warn ts=2026-05-10T18:57:24.13360071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:57:29.134216157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:57:31.526779932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:57:31.536731992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T18:57:33.422596621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55402"
level=warn ts=2026-05-10T18:57:34.134346688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T18:57:39.134261336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:57:41.119286126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T18:57:44.134362829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:57:48.885409802Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T18:57:49.117051087Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:57:49.117510648Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:57:49.133993738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:57:49.134012123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:57:49.134003464Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T18:57:49.134019856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T18:57:49.134054991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:57:49.134173217Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-10T18:57:49.134182584Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T18:57:49.134200501Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.4µs success=true
level=debug ts=2026-05-10T18:57:49.13439225Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-05-10T18:57:53.104281936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:57:53.489611564Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T18:57:54.133826038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:57:57.844705979Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T18:57:59.134092335Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:02.199941143Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=warn ts=2026-05-10T18:58:04.134023801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:06.554386513Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-05-10T18:58:09.13377138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:10.9088821Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-10T18:58:14.134341282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:15.263620625Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T18:58:17.671841852Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54946"
ts=2026-05-10T18:58:18.887694588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:58:19.134005787Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:58:19.134016464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:19.134031937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:58:19.134041709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:58:19.619090263Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T18:58:22.997356221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:58:23.007210105Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T18:58:23.608902415Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:58:23.973932532Z caller=checkpoint.go:372 msg="writing series" size="3.8 kB"
level=warn ts=2026-05-10T18:58:24.133801003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:28.328220941Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-05-10T18:58:29.133607939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:32.683869312Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=debug ts=2026-05-10T18:58:32.683893384Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=11
ts=2026-05-10T18:58:33.427571326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45792"
level=warn ts=2026-05-10T18:58:34.133928572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:37.037934163Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-10T18:58:38.095382939Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T18:58:38.097145766Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T18:58:39.13417253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:41.392955067Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-10T18:58:44.133878393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:58:44.900251614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37950"
level=debug ts=2026-05-10T18:58:45.747514693Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-05-10T18:58:47.674219736Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:55080"
ts=2026-05-10T18:58:48.88996939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T18:58:49.117303073Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:58:49.117807469Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:58:49.134211728Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:58:49.134211873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:49.13422654Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:58:49.134233853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:58:50.102779358Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T18:58:54.133531726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:58:54.457503617Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-10T18:58:55.489624292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T18:58:58.812582948Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-10T18:58:59.133584357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:03.167233107Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=warn ts=2026-05-10T18:59:04.133799347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:07.522279887Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T18:59:09.133620732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:11.877134921Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=debug ts=2026-05-10T18:59:13.289140338Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T18:59:14.133680768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T18:59:14.903177202Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49866"
level=debug ts=2026-05-10T18:59:16.232249424Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-10T18:59:18.892507782Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=warn ts=2026-05-10T18:59:19.134266474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T18:59:19.134286896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T18:59:19.134312668Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:59:19.134324515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:59:20.592941417Z caller=checkpoint.go:372 msg="writing series" size="6.7 MB"
level=debug ts=2026-05-10T18:59:20.592963311Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.8 MB" series=11
level=info ts=2026-05-10T18:59:21.691016656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T18:59:21.699271557Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T18:59:24.134398716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:24.941716537Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-05-10T18:59:24.941738917Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=1
level=debug ts=2026-05-10T18:59:27.08593752Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:59:29.133391847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:29.295889999Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
ts=2026-05-10T18:59:33.433692314Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:51690"
level=debug ts=2026-05-10T18:59:33.651430556Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-10T18:59:34.133776275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:38.006101359Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-10T18:59:39.133811663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:42.361481814Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T18:59:42.361506303Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=warn ts=2026-05-10T18:59:44.133823699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:46.715366127Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
ts=2026-05-10T18:59:48.895192498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T18:59:49.11794143Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T18:59:49.11903181Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T18:59:49.134336775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T18:59:49.134345964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:49.134360349Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T18:59:49.134371093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T18:59:51.070338836Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=debug ts=2026-05-10T18:59:53.328826054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T18:59:54.133623921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:55.425050559Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-10T18:59:59.133664126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T18:59:59.780374962Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-05-10T19:00:03.743432648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:00:04.133945741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:04.135054569Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-10T19:00:08.489706062Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T19:00:09.133996354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:12.845036339Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-10T19:00:14.133444754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:00:16.336074292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:00:16.345372522Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:00:17.199501086Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
ts=2026-05-10T19:00:18.897768156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:00:19.134281519Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:00:19.134329205Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:00:19.134340501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:00:19.134358823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:21.553975716Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T19:00:24.134424367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:25.909130692Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=debug ts=2026-05-10T19:00:29.124121412Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:00:29.124180097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:00:29.133390127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-10T19:00:29.133732624Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:00:30.264268774Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-05-10T19:00:33.438557947Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41634"
level=warn ts=2026-05-10T19:00:34.13362611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:34.618339178Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T19:00:38.973654443Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=debug ts=2026-05-10T19:00:38.973677127Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=13
level=warn ts=2026-05-10T19:00:39.13356572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:00:42.811196345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:00:42.83050796Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:00:43.327997849Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-05-10T19:00:44.133870895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:47.682896669Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
ts=2026-05-10T19:00:47.682898934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34360"
ts=2026-05-10T19:00:48.900945853Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:00:49.117201874Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:00:49.117683399Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T19:00:49.118986612Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T19:00:49.11902665Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976043 n=1
level=debug ts=2026-05-10T19:00:49.119067763Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976043 n=1
level=info ts=2026-05-10T19:00:49.134254809Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:00:49.134291907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:00:49.134303054Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:00:49.134337949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:52.03766363Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T19:00:54.133831911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:00:56.393034906Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-10T19:00:56.690728168Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:00:59.134169969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:00.748277395Z caller=checkpoint.go:372 msg="writing series" size="299 kB"
level=warn ts=2026-05-10T19:01:04.133939524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:04.835585332Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:01:05.102411421Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:01:09.133422037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:01:09.136511518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:01:09.144614822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:01:09.16660991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:01:09.173393054Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:01:09.457537703Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=debug ts=2026-05-10T19:01:13.812653401Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T19:01:14.133884287Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:18.167589288Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-05-10T19:01:18.903620576Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:01:19.13406566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:01:19.134078233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:19.134099965Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:01:19.134113632Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:01:22.521616183Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-10T19:01:24.133713419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:26.87751023Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-05-10T19:01:28.808234427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:01:29.133500161Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:31.232295894Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-10T19:01:34.133895045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:35.58631656Z caller=checkpoint.go:372 msg="writing series" size="227 kB"
level=debug ts=2026-05-10T19:01:35.586343125Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-05-10T19:01:39.134368962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:39.94198519Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=warn ts=2026-05-10T19:01:44.134162899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:44.296572204Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
ts=2026-05-10T19:01:44.911917194Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47088"
ts=2026-05-10T19:01:47.687699477Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54696"
level=debug ts=2026-05-10T19:01:48.651307183Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
ts=2026-05-10T19:01:48.90689023Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:01:49.117973311Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:01:49.118701212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:01:49.134094883Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:01:49.134124586Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:01:49.134132424Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:01:49.134144699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:01:50.462903745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:01:50.479695206Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:01:53.006054094Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-10T19:01:54.133451127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:01:57.360565224Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T19:01:59.134279574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:01:59.170369385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:01:59.178073895Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:02:01.715357501Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-05-10T19:02:03.728883396Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:02:04.133742614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:02:04.920241257Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:02:06.070373826Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-05-10T19:02:09.134305849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:02:10.425688593Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-05-10T19:02:10.425713765Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-05-10T19:02:14.133773783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:02:14.780534729Z caller=checkpoint.go:372 msg="writing series" size="533 kB"
ts=2026-05-10T19:02:18.909580315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:02:19.134692653Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:02:19.134729726Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:02:19.134740692Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:02:19.134835302Z caller=checkpoint.go:384 msg="flushing series" totalSize="533 kB" series=1
level=warn ts=2026-05-10T19:02:19.134873151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:02:19.138306116Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-05-10T19:02:19.139666172Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005322957s
level=warn ts=2026-05-10T19:02:24.134482944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:02:29.133841473Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:02:34.133666704Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:02:35.426347215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:02:35.436312447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:02:36.318090567Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:02:39.133568156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:02:44.134498634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:02:48.913242718Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:02:49.117463279Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:02:49.118278291Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:02:49.135708452Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:02:49.135746521Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:02:49.135882047Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-10T19:02:49.135893068Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:02:49.135916208Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.48µs success=true
level=debug ts=2026-05-10T19:02:49.136176987Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:02:49.136366211Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:02:49.136384716Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:02:49.136393455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:02:52.554623229Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-05-10T19:02:52.914273123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:02:52.915778988Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:02:53.030059864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:02:53.037344872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:02:54.134509877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:02:55.972576204Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T19:02:59.134732202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:02:59.39017847Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-10T19:03:02.807361505Z caller=checkpoint.go:372 msg="writing series" size="432 kB"
ts=2026-05-10T19:03:03.449454526Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60280"
level=warn ts=2026-05-10T19:03:04.134094236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:05.848822774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:03:06.225445901Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-10T19:03:09.133742671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:09.642902328Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-10T19:03:13.061200709Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-10T19:03:14.134206471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:16.478744941Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:03:18.916685901Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:03:19.133711289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:03:19.133742713Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:03:19.133756394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:03:19.133752404Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:19.896580355Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-10T19:03:23.314172393Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-10T19:03:23.314200616Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-05-10T19:03:24.133602845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:26.731580472Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-05-10T19:03:29.133680688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:30.149720822Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
ts=2026-05-10T19:03:33.452487923Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56910"
level=debug ts=2026-05-10T19:03:33.567110768Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-10T19:03:33.803559676Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:03:33.996142161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:03:34.004960307Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:03:34.133855198Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:34.480346202Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:03:36.985229975Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-10T19:03:39.13373102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:40.402807092Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-10T19:03:43.360716083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:03:43.368472979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:03:43.368514461Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:03:43.368543117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:03:43.375748662Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:03:43.820424115Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T19:03:44.133506472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:47.237808151Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:03:48.919869423Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:03:49.117202798Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:03:49.11754172Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:03:49.133994517Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:03:49.134020787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:03:49.134032922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:03:49.13404143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:50.655804538Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-05-10T19:03:54.073437597Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
level=warn ts=2026-05-10T19:03:54.1337346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:03:57.490940238Z caller=checkpoint.go:372 msg="writing series" size="138 kB"
level=debug ts=2026-05-10T19:03:57.490965727Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=10
level=info ts=2026-05-10T19:03:58.337589033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:03:58.348541206Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:03:59.134233395Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:00.601856282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:04:00.908758089Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-10T19:04:03.455080182Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:32996"
level=warn ts=2026-05-10T19:04:04.133466713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:04.326352811Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-10T19:04:07.744054069Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-10T19:04:09.134174531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:11.166153627Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-05-10T19:04:11.166178561Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.6 MB" series=4
level=warn ts=2026-05-10T19:04:14.134076141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:14.580884448Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-05-10T19:04:14.580906824Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=1
ts=2026-05-10T19:04:17.69828338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57896"
level=debug ts=2026-05-10T19:04:17.997450512Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
ts=2026-05-10T19:04:18.922334466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=warn ts=2026-05-10T19:04:19.133746214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:04:19.133785567Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:04:19.133797622Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:04:19.133807255Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:04:20.476611852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:04:20.489175718Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:04:21.414976959Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-05-10T19:04:24.134003099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:04:24.610541523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:04:24.620593121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:04:24.833222496Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=debug ts=2026-05-10T19:04:28.250224343Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-10T19:04:28.899426424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:04:29.13379544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:31.668309091Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-10T19:04:34.134384597Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:35.086232525Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-05-10T19:04:35.490624021Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:04:38.514177547Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-10T19:04:38.514201163Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=7
level=warn ts=2026-05-10T19:04:39.133533932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:41.921759595Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-10T19:04:44.133496877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:04:44.924437396Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55858"
level=debug ts=2026-05-10T19:04:45.339140688Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=debug ts=2026-05-10T19:04:48.75750099Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
ts=2026-05-10T19:04:48.925119238Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:04:49.117412952Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:04:49.118854422Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:04:49.134286435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:04:49.134325434Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:04:49.134336348Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:04:49.134363934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:52.174364767Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=warn ts=2026-05-10T19:04:54.134061009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:55.592102591Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-10T19:04:59.010573989Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-10T19:04:59.133569421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:04:59.99231486Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:05:02.427473695Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T19:05:04.134473756Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:05.84571144Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=info ts=2026-05-10T19:05:08.377038777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:05:08.389126554Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:05:09.065199943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:05:09.073102004Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:05:09.133460007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:09.262850052Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=debug ts=2026-05-10T19:05:12.680784397Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-10T19:05:14.133736658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:16.098703037Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:05:18.928431031Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:05:19.134351555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:05:19.134352762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:19.134379688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:05:19.134388798Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:05:19.516225452Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-05-10T19:05:19.516250518Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-10T19:05:22.93439461Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:05:24.134315191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:25.522162971Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:05:26.351546617Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T19:05:29.134233312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:29.770090466Z caller=checkpoint.go:372 msg="writing series" size="214 kB"
level=debug ts=2026-05-10T19:05:30.107997597Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:05:33.187235672Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-10T19:05:34.134143606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:36.605355531Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-10T19:05:39.134427971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:40.022692719Z caller=checkpoint.go:372 msg="writing series" size="117 kB"
level=debug ts=2026-05-10T19:05:43.440795399Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-10T19:05:44.133895876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:46.858626636Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-10T19:05:48.930964456Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:05:49.117012616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:05:49.118945672Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:05:49.134246897Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:05:49.134264662Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:05:49.134273753Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:05:49.134371927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:05:49.556839016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:05:49.566492739Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:05:50.276230143Z caller=checkpoint.go:372 msg="writing series" size="6.5 kB"
level=debug ts=2026-05-10T19:05:53.693585867Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-10T19:05:54.133737054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:05:56.959322355Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:05:57.110988687Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T19:05:59.13366322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:00.528991858Z caller=checkpoint.go:372 msg="writing series" size="423 kB"
level=info ts=2026-05-10T19:06:02.12767588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:06:02.136657429Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:06:03.466067838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56360"
level=debug ts=2026-05-10T19:06:03.947141351Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:06:04.133814668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:07.364407079Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-10T19:06:09.133896795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:10.781801671Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:06:14.133383776Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:14.199568393Z caller=checkpoint.go:372 msg="writing series" size="9.6 kB"
level=debug ts=2026-05-10T19:06:15.477678578Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:06:17.61771031Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-10T19:06:18.933574486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:06:19.133483791Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:06:19.133512004Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:06:19.133520895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:06:19.133576082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:21.03562939Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=debug ts=2026-05-10T19:06:23.164746053Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:06:24.134228254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:24.453275176Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-05-10T19:06:27.871231767Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-05-10T19:06:27.871254014Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=20
level=warn ts=2026-05-10T19:06:29.133406981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:06:30.721081126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:06:30.732337095Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:06:31.288315945Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=warn ts=2026-05-10T19:06:34.134322649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:34.706112352Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-05-10T19:06:38.124297898Z caller=checkpoint.go:372 msg="writing series" size="333 kB"
level=warn ts=2026-05-10T19:06:39.133587942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:41.541357357Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=info ts=2026-05-10T19:06:43.544524267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:06:43.557312061Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:06:43.557335052Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:06:43.557366394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:06:43.56388394Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:06:44.133852728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:44.959168824Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-05-10T19:06:47.611540009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:06:48.376774639Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
ts=2026-05-10T19:06:48.936573473Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:06:49.116974354Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:06:49.117324851Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:06:49.133711115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:06:49.133722551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:06:49.133736713Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:06:49.133757787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:06:51.794795132Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-05-10T19:06:51.794817985Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-10T19:06:54.134352573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:06:54.619754226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:06:54.62835049Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:06:55.213138202Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-10T19:06:58.629877938Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T19:06:59.13349943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:07:02.047742981Z caller=checkpoint.go:372 msg="writing series" size="465 kB"
level=warn ts=2026-05-10T19:07:04.133480907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:07:05.466059221Z caller=checkpoint.go:372 msg="writing series" size="222 kB"
level=debug ts=2026-05-10T19:07:08.88406691Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-10T19:07:09.133463063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:07:10.926167063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:07:10.933339106Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:07:12.302046082Z caller=checkpoint.go:372 msg="writing series" size="690 kB"
level=debug ts=2026-05-10T19:07:12.302069671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-10T19:07:12.499910081Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:07:12.501966484Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T19:07:14.13451128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:07:14.934135742Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47546"
level=debug ts=2026-05-10T19:07:15.719084263Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-05-10T19:07:17.38184954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:07:17.389768133Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:07:17.709998472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35774"
ts=2026-05-10T19:07:18.939418336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:07:19.1337406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:07:19.13377296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:07:19.133785082Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:07:19.133788534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:07:19.137009065Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 kB" series=1
level=info ts=2026-05-10T19:07:19.235627065Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-05-10T19:07:19.240680885Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.10458563s
level=debug ts=2026-05-10T19:07:21.754228795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:07:24.135692072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:07:29.134097813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:07:33.473604252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41034"
level=warn ts=2026-05-10T19:07:34.136702935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:07:39.134867781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:07:44.133624735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:07:47.712560913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34700"
level=debug ts=2026-05-10T19:07:48.923604371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:07:48.945373722Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:07:49.117775102Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:07:49.118353863Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:07:49.137834229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:07:49.137890845Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:07:49.137971157Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:07:49.137992801Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:07:49.138002576Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:07:49.138033336Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-10T19:07:49.138042169Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:07:49.138064793Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.834µs success=true
level=debug ts=2026-05-10T19:07:49.138338029Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-05-10T19:07:52.557515828Z caller=checkpoint.go:372 msg="writing series" size="296 kB"
level=warn ts=2026-05-10T19:07:54.136713829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:07:55.299169618Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:07:55.974356114Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T19:07:59.143743547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:07:59.392223029Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-10T19:08:02.810598714Z caller=checkpoint.go:372 msg="writing series" size="702 kB"
level=debug ts=2026-05-10T19:08:02.810624173Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-05-10T19:08:03.477118872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:32842"
level=warn ts=2026-05-10T19:08:04.135712509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:08:05.380471493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:08:05.389787339Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:08:06.227995495Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-10T19:08:06.242198407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:08:06.27074604Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:08:09.150742708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:09.646744745Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-10T19:08:13.066806198Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-10T19:08:14.136830361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:16.480837485Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:08:17.715209887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:32788"
ts=2026-05-10T19:08:18.956719433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:08:19.135442053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:08:19.135476225Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:08:19.135487491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:08:19.135513919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:19.898262574Z caller=checkpoint.go:372 msg="writing series" size="4.5 kB"
level=debug ts=2026-05-10T19:08:23.316534902Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
level=warn ts=2026-05-10T19:08:24.144750749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:24.228987589Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:08:26.73537372Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-10T19:08:29.142724442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:30.151756035Z caller=checkpoint.go:372 msg="writing series" size="3.6 kB"
level=debug ts=2026-05-10T19:08:33.568814466Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-10T19:08:34.133988078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:36.987261273Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-10T19:08:39.139009797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:39.737945203Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:08:40.405078817Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-10T19:08:40.405103228Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=debug ts=2026-05-10T19:08:43.822237309Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T19:08:44.134259638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:47.239811027Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:08:47.7174261Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40732"
ts=2026-05-10T19:08:48.960986548Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:08:49.117028803Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:08:49.117799627Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:08:49.13469802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:08:49.134736748Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:08:49.134746339Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:08:49.134742518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:08:50.657883595Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-05-10T19:08:54.07595192Z caller=checkpoint.go:372 msg="writing series" size="304 kB"
level=warn ts=2026-05-10T19:08:54.133747699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:08:55.35854355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:08:55.366232926Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:08:57.493160812Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-10T19:08:58.873355853Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:08:59.133545181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:00.911081777Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-10T19:09:02.169870168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:09:02.178197039Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:09:04.139176738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:04.328784594Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=debug ts=2026-05-10T19:09:04.328808087Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-10T19:09:07.746621055Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-05-10T19:09:09.134994359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:11.168261322Z caller=checkpoint.go:372 msg="writing series" size="8.1 MB"
level=debug ts=2026-05-10T19:09:11.168285324Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.1 MB" series=2
level=warn ts=2026-05-10T19:09:14.135692231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:14.583162927Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-10T19:09:14.583207227Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=1
ts=2026-05-10T19:09:17.719827231Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54218"
level=debug ts=2026-05-10T19:09:17.999839018Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-10T19:09:18.963811278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:09:19.134949875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:09:19.13499774Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:09:19.135009028Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:09:19.135001832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:21.417746222Z caller=checkpoint.go:372 msg="writing series" size="370 kB"
level=debug ts=2026-05-10T19:09:21.572512542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:09:23.546214226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:09:24.133851183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:24.835029072Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-05-10T19:09:28.252897072Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-10T19:09:29.133881499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:31.670170593Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-10T19:09:34.135745952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:35.088123111Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-10T19:09:37.596421292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:09:37.606436593Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:09:37.606465745Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:09:37.606493801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:09:37.614277811Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:09:38.518218096Z caller=checkpoint.go:372 msg="writing series" size="21 MB"
level=debug ts=2026-05-10T19:09:38.518240945Z caller=checkpoint.go:384 msg="flushing series" totalSize="22 MB" series=7
level=warn ts=2026-05-10T19:09:39.136834353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:41.923700709Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-10T19:09:44.138529357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:09:44.943571464Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:52304"
level=debug ts=2026-05-10T19:09:45.34197478Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
ts=2026-05-10T19:09:47.722845136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46120"
level=info ts=2026-05-10T19:09:47.936784762Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:09:47.954560734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:09:48.420923166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:09:48.428971864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:09:48.759225351Z caller=checkpoint.go:372 msg="writing series" size="278 kB"
ts=2026-05-10T19:09:48.967072486Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:09:49.117047875Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:09:49.11704686Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:09:49.133666448Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:09:49.133689657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:09:49.133700608Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:09:49.134746901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:52.177119465Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=warn ts=2026-05-10T19:09:54.134726351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:09:55.400689696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:09:55.594437426Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-05-10T19:09:59.012549849Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-05-10T19:09:59.137803934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:02.429883707Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-10T19:10:03.487011235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39240"
level=warn ts=2026-05-10T19:10:04.134117082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:04.881652027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:10:05.847742438Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T19:10:09.135068306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:09.265853499Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-05-10T19:10:09.265876478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-10T19:10:12.683656512Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-05-10T19:10:14.13671908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:10:14.184874123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:10:14.194473373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:10:16.10114899Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:10:18.969945737Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=warn ts=2026-05-10T19:10:19.13670627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:10:19.137794106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:10:19.137818219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:10:19.137830131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:10:19.518462049Z caller=checkpoint.go:372 msg="writing series" size="822 kB"
level=debug ts=2026-05-10T19:10:22.936846987Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:10:24.136705502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:26.354268555Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T19:10:29.136804655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:29.771380916Z caller=checkpoint.go:372 msg="writing series" size="263 kB"
level=debug ts=2026-05-10T19:10:29.771403657Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=debug ts=2026-05-10T19:10:31.279433934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:10:33.189100863Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-10T19:10:33.448876747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:10:33.463913493Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:10:34.136283615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:36.607009999Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-10T19:10:39.139163938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:40.025592345Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-10T19:10:43.44283898Z caller=checkpoint.go:372 msg="writing series" size="320 kB"
level=warn ts=2026-05-10T19:10:44.133988166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:46.860905923Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-10T19:10:48.729838431Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:10:48.732213149Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-05-10T19:10:48.973095123Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:10:49.117194081Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:10:49.117227167Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:10:49.133787202Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:10:49.133846934Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:10:49.133858213Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:10:49.13399499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:50.278488303Z caller=checkpoint.go:372 msg="writing series" size="7.7 kB"
level=debug ts=2026-05-10T19:10:53.696321041Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=warn ts=2026-05-10T19:10:54.135358179Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:10:57.113496022Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-10T19:10:58.524887563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:10:58.53522137Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:10:59.134713178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:00.531222952Z caller=checkpoint.go:372 msg="writing series" size="589 kB"
level=debug ts=2026-05-10T19:11:00.531248604Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=debug ts=2026-05-10T19:11:00.923171021Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:11:03.954760101Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:11:04.141756647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:07.36714717Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-05-10T19:11:09.133939697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:10.784587608Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:11:14.134166951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:14.202339777Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
ts=2026-05-10T19:11:14.948629048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40728"
level=debug ts=2026-05-10T19:11:17.619706593Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-10T19:11:17.729813401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39930"
ts=2026-05-10T19:11:18.975786325Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:11:19.133884892Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:11:19.133932754Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:11:19.133944171Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:11:19.133966311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:21.03805072Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-05-10T19:11:24.133539443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:24.455093738Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-05-10T19:11:26.958356349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="188 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:11:26.970001011Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:11:27.872992095Z caller=checkpoint.go:372 msg="writing series" size="308 kB"
level=warn ts=2026-05-10T19:11:29.13423903Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:31.29109648Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=warn ts=2026-05-10T19:11:34.133803881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:34.708646487Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
level=debug ts=2026-05-10T19:11:35.018559692Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:11:37.243765007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:11:38.126940245Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=debug ts=2026-05-10T19:11:38.126962959Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=11
level=warn ts=2026-05-10T19:11:39.133825569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:41.544187262Z caller=checkpoint.go:372 msg="writing series" size="309 kB"
level=warn ts=2026-05-10T19:11:44.134325812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:11:44.951876484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:58186"
level=debug ts=2026-05-10T19:11:44.961477852Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
ts=2026-05-10T19:11:47.732341246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39180"
level=debug ts=2026-05-10T19:11:48.378950036Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
ts=2026-05-10T19:11:48.978237724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:11:49.117837652Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:11:49.118694628Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:11:49.134184407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:11:49.134217612Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T19:11:49.134207188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:11:49.134229327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:11:51.796804298Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-10T19:11:54.002131451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:11:54.010699338Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:11:54.13441454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:55.2146002Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-10T19:11:58.632041654Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T19:11:59.133822445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:11:59.736708256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:12:02.050988352Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:12:02.051014292Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=warn ts=2026-05-10T19:12:04.134035196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:12:05.468280111Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-05-10T19:12:08.885802773Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=warn ts=2026-05-10T19:12:09.134147646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:12:12.303460178Z caller=checkpoint.go:372 msg="writing series" size="640 kB"
level=warn ts=2026-05-10T19:12:14.136912536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:12:14.955280078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37498"
level=debug ts=2026-05-10T19:12:15.721030812Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:12:15.721054676Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-10T19:12:18.980726617Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:12:19.139756957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:12:19.139790239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:12:19.13979983Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:12:19.139852239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:12:19.145814471Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-05-10T19:12:19.147202841Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008948357s
level=warn ts=2026-05-10T19:12:24.137787178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:12:26.005425137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="212 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:12:26.015338542Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:12:29.133782923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:12:30.085341729Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:12:32.663102461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:12:34.133841197Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:12:39.133755947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:12:43.838972942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:12:43.848292486Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:12:44.134151278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:12:44.958183389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45408"
ts=2026-05-10T19:12:48.983872767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:12:49.117004388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:12:49.117430925Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:12:49.134269483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:12:49.134287237Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:12:49.134309632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:12:49.134319326Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:12:49.13433165Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:12:49.134867543Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-10T19:12:49.134881363Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:12:49.134910537Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=21.645µs success=true
level=debug ts=2026-05-10T19:12:49.135254907Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-10T19:12:52.510961465Z caller=checkpoint.go:372 msg="writing series" size="689 kB"
level=warn ts=2026-05-10T19:12:54.134151961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:12:55.885568415Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=warn ts=2026-05-10T19:12:59.133891026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:12:59.26119587Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=debug ts=2026-05-10T19:13:02.636541741Z caller=checkpoint.go:372 msg="writing series" size="897 kB"
level=debug ts=2026-05-10T19:13:02.636572415Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=5
level=warn ts=2026-05-10T19:13:04.133900542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:04.207223822Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:13:06.010799989Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-10T19:13:09.13364527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:09.38572063Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-05-10T19:13:12.760997526Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-10T19:13:14.133769644Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:15.178809447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:13:16.135766829Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:13:17.740384027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52246"
ts=2026-05-10T19:13:18.986328736Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=warn ts=2026-05-10T19:13:19.133386209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:13:19.133796206Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:13:19.133818661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:13:19.133828645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:13:19.155166218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="235 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:13:19.162566327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:13:19.162586707Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:13:19.162614429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="235 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:13:19.169150644Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:13:19.510450758Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=debug ts=2026-05-10T19:13:22.885973534Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=warn ts=2026-05-10T19:13:24.133794772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:26.260508921Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-05-10T19:13:28.955851755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:13:28.963555834Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:13:29.133914304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:29.63583908Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-10T19:13:29.87461176Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:13:33.010262882Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
ts=2026-05-10T19:13:33.502574067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41256"
level=warn ts=2026-05-10T19:13:34.133979495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:13:34.930292206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="259 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:13:34.939052268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:13:36.38546098Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-10T19:13:39.133821634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:39.760979568Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-10T19:13:43.137505046Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-10T19:13:44.134724035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:13:44.962679837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46450"
level=debug ts=2026-05-10T19:13:46.510927874Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:13:47.742757123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52088"
ts=2026-05-10T19:13:48.989500654Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:13:49.117923806Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:13:49.118694436Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:13:49.134073122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:13:49.134081128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:13:49.134107855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:13:49.134119993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:13:49.88942481Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=debug ts=2026-05-10T19:13:49.889454077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=14
level=debug ts=2026-05-10T19:13:53.260981606Z caller=checkpoint.go:372 msg="writing series" size="438 kB"
level=warn ts=2026-05-10T19:13:54.134036324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:13:56.635534501Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=debug ts=2026-05-10T19:13:58.302180941Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:13:59.133814914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:00.011261417Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=debug ts=2026-05-10T19:14:03.385422469Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=debug ts=2026-05-10T19:14:03.385447984Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
ts=2026-05-10T19:14:03.505574415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45192"
level=warn ts=2026-05-10T19:14:04.133863008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:06.760498437Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-10T19:14:08.3570849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="259 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:14:08.369028181Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:14:09.133646867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:10.146737745Z caller=checkpoint.go:372 msg="writing series" size="8.9 MB"
level=debug ts=2026-05-10T19:14:10.146774028Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.0 MB" series=2
level=debug ts=2026-05-10T19:14:13.007762238Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:14:13.511529295Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T19:14:13.511555492Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=1
level=warn ts=2026-05-10T19:14:14.134413135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:16.886629863Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-10T19:14:16.886675615Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
ts=2026-05-10T19:14:17.745553029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47558"
ts=2026-05-10T19:14:18.992549348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:14:19.133681773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:14:19.133715201Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:14:19.133724044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:14:19.133726721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:14:19.755306261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:14:19.764081101Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:14:20.26105826Z caller=checkpoint.go:372 msg="writing series" size="283 kB"
level=debug ts=2026-05-10T19:14:23.635341221Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=warn ts=2026-05-10T19:14:24.133957934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:27.010716787Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-10T19:14:29.134081702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:29.71115771Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:14:30.38615912Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-10T19:14:33.760613512Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=warn ts=2026-05-10T19:14:34.134218204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:37.178003686Z caller=checkpoint.go:372 msg="writing series" size="36 MB"
level=debug ts=2026-05-10T19:14:37.17803138Z caller=checkpoint.go:384 msg="flushing series" totalSize="36 MB" series=6
level=warn ts=2026-05-10T19:14:39.134786021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:40.5104492Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=debug ts=2026-05-10T19:14:43.886124111Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=warn ts=2026-05-10T19:14:44.134486005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:47.260861545Z caller=checkpoint.go:372 msg="writing series" size="273 kB"
ts=2026-05-10T19:14:48.995850301Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:14:49.117498371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:14:49.118191152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:14:49.134741038Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:14:49.134784236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:14:49.134798142Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:14:49.134808378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:14:50.025229772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:14:50.046901131Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:14:50.63591728Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-05-10T19:14:54.011111489Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T19:14:54.13774892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:14:57.385591241Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-05-10T19:14:58.546369826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:14:58.548886337Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T19:14:59.136798954Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:00.760404038Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-10T19:15:03.510385529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35228"
level=warn ts=2026-05-10T19:15:04.136204175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:04.136213905Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-10T19:15:04.136236659Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=debug ts=2026-05-10T19:15:04.210538104Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:15:04.21060487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:15:04.218619084Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:15:07.511163332Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-05-10T19:15:09.133679294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:10.885349534Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:15:14.13412705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:14.260654115Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=info ts=2026-05-10T19:15:14.951578984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:15:14.962974462Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T19:15:14.968748826Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:36532"
level=debug ts=2026-05-10T19:15:17.637398552Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:15:17.750797484Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:41782"
ts=2026-05-10T19:15:18.999425656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:15:19.133740911Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:15:19.133776359Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:15:19.133786894Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:15:19.133808676Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:15:20.148715037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:15:20.162743689Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:15:21.012158585Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-10T19:15:21.012196392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=warn ts=2026-05-10T19:15:24.13430775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:24.385407869Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-05-10T19:15:27.76112036Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-05-10T19:15:29.133722319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:31.135418086Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=info ts=2026-05-10T19:15:33.143957996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="282 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:15:33.154925696Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:15:33.513415378Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54570"
level=warn ts=2026-05-10T19:15:34.134008821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:34.337876412Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:15:34.337935891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:15:34.348443756Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:15:34.511021981Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=debug ts=2026-05-10T19:15:37.886154312Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-10T19:15:39.137533483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:41.26112871Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=warn ts=2026-05-10T19:15:44.133727192Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:44.63613386Z caller=checkpoint.go:372 msg="writing series" size="330 kB"
level=debug ts=2026-05-10T19:15:44.636164845Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-10T19:15:47.637795526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:15:47.650527248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:15:47.753029348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44490"
level=debug ts=2026-05-10T19:15:48.010360749Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-10T19:15:49.002209621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:15:49.116985947Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:15:49.116983351Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T19:15:49.118313005Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T19:15:49.118357733Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976044 n=1
level=debug ts=2026-05-10T19:15:49.118404181Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976044 n=1
level=info ts=2026-05-10T19:15:49.133468145Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:15:49.133494779Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:15:49.133505976Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:15:49.133496648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:15:50.419045284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:15:50.43007015Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:15:51.385411338Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
level=debug ts=2026-05-10T19:15:53.127564435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:15:54.134330459Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:15:54.760476168Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-10T19:15:58.135399932Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T19:15:59.133871383Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:01.510910353Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=warn ts=2026-05-10T19:16:04.14575838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:04.885747724Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-05-10T19:16:08.261245632Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-10T19:16:08.630286846Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:16:09.138513789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:11.637755952Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-10T19:16:12.263200453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:16:12.27253715Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:16:12.272561669Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:16:12.272588232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:16:12.280358482Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:16:14.133627332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:15.010852668Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=info ts=2026-05-10T19:16:16.808144627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="306 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:16:16.817025405Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:16:17.756373875Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57878"
level=debug ts=2026-05-10T19:16:18.385232462Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
ts=2026-05-10T19:16:19.005591395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=warn ts=2026-05-10T19:16:19.133590379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:16:19.133768989Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:16:19.133789465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:16:19.133800093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:16:21.76094044Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-05-10T19:16:24.133934541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:16:24.997108067Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:16:25.006921496Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:16:25.135663393Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-10T19:16:27.27186957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:16:27.27985704Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:16:28.51072582Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-05-10T19:16:29.134270201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:31.886317385Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=debug ts=2026-05-10T19:16:31.886343034Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-05-10T19:16:34.133905843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:35.260988773Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:16:38.150347936Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:16:38.637071084Z caller=checkpoint.go:372 msg="writing series" size="539 kB"
level=warn ts=2026-05-10T19:16:39.143247498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:42.010444708Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:16:42.010465725Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-10T19:16:44.139729329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:44.218767759Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T19:16:44.975216928Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:54368"
level=debug ts=2026-05-10T19:16:45.386082238Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-10T19:16:46.954990418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:16:46.962318044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:16:47.759482667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53164"
level=debug ts=2026-05-10T19:16:48.761180823Z caller=checkpoint.go:372 msg="writing series" size="127 kB"
ts=2026-05-10T19:16:49.008920343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T19:16:49.116959666Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:16:49.118004462Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-05-10T19:16:49.133382623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:16:49.133390452Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:16:49.133411763Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:16:49.133421653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:16:51.098582669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:16:51.106089724Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:16:52.136807012Z caller=checkpoint.go:372 msg="writing series" size="277 kB"
level=warn ts=2026-05-10T19:16:54.13757149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:16:55.510935103Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-05-10T19:16:58.885514488Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-05-10T19:16:59.138780275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:02.261753766Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-10T19:17:02.261778605Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=6
level=warn ts=2026-05-10T19:17:04.145665703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:05.642608879Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-05-10T19:17:07.615779543Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:17:09.010369382Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T19:17:09.137346997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:17:11.940197325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="306 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:17:11.954170346Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:17:12.385623141Z caller=checkpoint.go:372 msg="writing series" size="694 kB"
level=warn ts=2026-05-10T19:17:14.133436346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:15.761220758Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:17:15.761244016Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
ts=2026-05-10T19:17:19.011598844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:17:19.133684953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:17:19.133690109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:19.133713848Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:17:19.133721539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:17:19.143738929Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-05-10T19:17:19.146191286Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.01109268s
level=warn ts=2026-05-10T19:17:24.134026558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:17:25.593533178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:17:25.604175083Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:17:29.133680483Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:17:34.13400521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:17:38.283481724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:17:38.292760883Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:17:39.133970005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:39.867477999Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:17:41.844980981Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:17:43.801231824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:17:43.810289676Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:17:44.133461312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:17:47.765934067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54638"
ts=2026-05-10T19:17:49.014757699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:17:49.11696866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:17:49.117479471Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:17:49.13395441Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:17:49.133979769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:17:49.133983658Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:17:49.13400302Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:17:49.134014105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:17:49.13411848Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-10T19:17:49.134126989Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:17:49.134153766Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=22.016µs success=true
level=debug ts=2026-05-10T19:17:49.134443264Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-10T19:17:52.468497346Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T19:17:54.133707273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:55.802040726Z caller=checkpoint.go:372 msg="writing series" size="878 kB"
level=debug ts=2026-05-10T19:17:55.802063545Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-10T19:17:56.231036533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:17:56.242246035Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:17:59.13369477Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:17:59.134837757Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-10T19:18:02.467931595Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-10T19:18:04.134139708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:05.802011134Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-10T19:18:05.802033214Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-10T19:18:08.230763628Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:18:09.133844202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:09.134970973Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-10T19:18:12.468311724Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:18:14.133752183Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:15.80139486Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T19:18:19.017225974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=warn ts=2026-05-10T19:18:19.133699581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:18:19.133710257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:18:19.133749168Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:18:19.133758261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:18:19.135010118Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-10T19:18:22.468095719Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-05-10T19:18:23.363764436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:18:23.375604099Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:18:24.134084841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:25.801629455Z caller=checkpoint.go:372 msg="writing series" size="425 kB"
level=warn ts=2026-05-10T19:18:29.133996526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:29.135287824Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-05-10T19:18:29.959492917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:18:29.969110996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:18:32.468394229Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-10T19:18:33.529757945Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:37426"
level=warn ts=2026-05-10T19:18:34.134031003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:18:35.664768788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:18:35.672829854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:18:35.801305687Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-10T19:18:35.80132777Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=debug ts=2026-05-10T19:18:35.996958487Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:18:39.133733847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:39.134823982Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-10T19:18:42.048847967Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:18:42.468279179Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-10T19:18:44.13372971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:45.801097344Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
ts=2026-05-10T19:18:47.771552624Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:41044"
ts=2026-05-10T19:18:49.020432569Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:18:49.116968487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:18:49.116966862Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:18:49.133398854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:18:49.13342124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:18:49.133431837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:18:49.133510579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:49.134602976Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-10T19:18:51.82947189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="330 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:18:51.839165012Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:18:52.468301164Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:18:54.133706532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:55.801912109Z caller=checkpoint.go:372 msg="writing series" size="380 kB"
level=debug ts=2026-05-10T19:18:55.801942626Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.0 MB" series=6
level=warn ts=2026-05-10T19:18:59.133572005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:18:59.135008209Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-10T19:19:02.46954979Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-05-10T19:19:04.133996978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:05.802070954Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
level=info ts=2026-05-10T19:19:06.399075983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:19:06.411176292Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:19:09.135030106Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-05-10T19:19:09.13507085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:10.222769079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:19:12.477007685Z caller=checkpoint.go:372 msg="writing series" size="9.5 MB"
level=debug ts=2026-05-10T19:19:12.477031886Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=5
level=warn ts=2026-05-10T19:19:14.133845003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:15.80291205Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-05-10T19:19:15.802940865Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=1
level=info ts=2026-05-10T19:19:16.6379038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:19:16.647652952Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:19:19.023760591Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:19:19.133710571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:19:19.133754866Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:19:19.133767491Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:19:19.133793396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:19.136185955Z caller=checkpoint.go:372 msg="writing series" size="3.2 MB"
level=debug ts=2026-05-10T19:19:19.136202763Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=1
level=debug ts=2026-05-10T19:19:22.468159931Z caller=checkpoint.go:372 msg="writing series" size="900 kB"
level=warn ts=2026-05-10T19:19:24.133947387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:25.801152284Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-05-10T19:19:25.952687572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:19:25.960904322Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:19:29.133675998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:29.134799584Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-05-10T19:19:29.13481654Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-10T19:19:32.468041521Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=info ts=2026-05-10T19:19:33.328752228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:19:33.337248879Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:19:33.337274291Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:19:33.337307763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:19:33.344770665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:19:34.133927427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:35.27880265Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:19:35.278857069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:19:35.283663805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:19:35.801101413Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=warn ts=2026-05-10T19:19:39.134368143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:39.171249026Z caller=checkpoint.go:372 msg="writing series" size="44 MB"
level=debug ts=2026-05-10T19:19:39.171275222Z caller=checkpoint.go:384 msg="flushing series" totalSize="44 MB" series=3
level=debug ts=2026-05-10T19:19:39.895020301Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:19:39.896647715Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T19:19:42.468498533Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:19:44.133891613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:19:44.988568046Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40286"
level=debug ts=2026-05-10T19:19:45.801277274Z caller=checkpoint.go:372 msg="writing series" size="367 kB"
ts=2026-05-10T19:19:49.027147735Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:19:49.117005167Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:19:49.117748355Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:19:49.134232493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:19:49.13425636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:19:49.134267914Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:19:49.134281594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:49.134441427Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-10T19:19:51.877156624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:19:51.885999241Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:19:51.903115969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:19:51.907453323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:19:52.468619908Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T19:19:52.957071089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:19:52.965426154Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:19:54.133896809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:55.80159636Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T19:19:59.134814607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:19:59.135087458Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-05-10T19:20:01.491292422Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:20:02.468022767Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-10T19:20:03.538685714Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57094"
level=warn ts=2026-05-10T19:20:04.133790893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:05.801663511Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T19:20:09.134197942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:09.136221537Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=debug ts=2026-05-10T19:20:09.136251512Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=9
level=info ts=2026-05-10T19:20:09.39948615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:20:09.414848256Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:20:12.467753542Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-10T19:20:12.964356991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:20:12.971492186Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:20:14.133894163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:15.801667357Z caller=checkpoint.go:372 msg="writing series" size="515 B"
ts=2026-05-10T19:20:17.777104201Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40774"
ts=2026-05-10T19:20:19.029852189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:20:19.136626047Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-05-10T19:20:19.136708608Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:20:19.136813478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:20:19.136827504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:20:19.1368368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:20:22.469873323Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-10T19:20:22.469905074Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=4
level=warn ts=2026-05-10T19:20:24.134194148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:20:25.039856944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:20:25.04873947Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:20:25.801080519Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:20:29.133775089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:29.134983889Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-10T19:20:31.226120787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:20:32.237777186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:20:32.24766564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:20:32.46887544Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-05-10T19:20:32.649317963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T19:20:33.541733281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41738"
level=warn ts=2026-05-10T19:20:34.133618067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:20:35.721899958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:20:35.730599315Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:20:35.801841024Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-05-10T19:20:39.133386312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:39.134388555Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T19:20:42.468099186Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
level=warn ts=2026-05-10T19:20:44.133992962Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:20:44.992518605Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:48424"
level=debug ts=2026-05-10T19:20:45.802295556Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T19:20:45.802318616Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
ts=2026-05-10T19:20:49.034431844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:20:49.117356142Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:20:49.11820253Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:20:49.133543092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:20:49.133560729Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:20:49.133568523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:20:49.133614946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:49.134702288Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-10T19:20:52.468285097Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=warn ts=2026-05-10T19:20:54.13442486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:55.801230438Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=warn ts=2026-05-10T19:20:59.134313135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:20:59.134404918Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-10T19:21:01.449004134Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:21:01.783657594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:21:01.796763099Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:21:02.468979913Z caller=checkpoint.go:372 msg="writing series" size="545 kB"
level=warn ts=2026-05-10T19:21:04.133743769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:21:04.603004041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:21:04.612106775Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:21:05.801426811Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:21:09.133568836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:09.135322238Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:21:09.135338457Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=7
level=debug ts=2026-05-10T19:21:12.468576677Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:21:14.133566442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:21:14.995373958Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51666"
level=debug ts=2026-05-10T19:21:15.801653895Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-05-10T19:21:16.421929109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:21:16.432729054Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
ts=2026-05-10T19:21:19.037322783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:21:19.133498016Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:21:19.133526627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:21:19.133535529Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:21:19.133543546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:19.134612753Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-10T19:21:22.060781447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:21:22.065609602Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:21:22.468188248Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-05-10T19:21:23.362266939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="354 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:21:23.370453447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:21:23.396548016Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:21:24.133516922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:25.801598795Z caller=checkpoint.go:372 msg="writing series" size="205 kB"
level=warn ts=2026-05-10T19:21:29.133708358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:29.135007904Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-10T19:21:31.185268072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:21:32.468384282Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-05-10T19:21:34.133906772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:35.801433254Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:21:35.801455737Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-10T19:21:39.133844524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:39.135688933Z caller=checkpoint.go:372 msg="writing series" size="802 kB"
level=debug ts=2026-05-10T19:21:42.468533662Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:21:42.46856341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-10T19:21:44.134475225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:21:45.31501572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:21:45.325903497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:21:45.801259048Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-10T19:21:49.040338842Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:21:49.117171379Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:21:49.117688448Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:21:49.134152033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:21:49.134179672Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:21:49.13418756Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:21:49.134244338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:49.135352291Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-10T19:21:52.468561993Z caller=checkpoint.go:372 msg="writing series" size="321 kB"
level=info ts=2026-05-10T19:21:53.234852094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:21:53.24273942Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:21:53.242763529Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:21:53.242788413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:21:53.249878952Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:21:54.133551624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:55.801713669Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-10T19:21:56.829317907Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:21:59.133903277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:21:59.134888876Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-10T19:22:02.469222819Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-10T19:22:02.469245015Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
ts=2026-05-10T19:22:03.547995725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55100"
level=debug ts=2026-05-10T19:22:03.857246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:22:04.134249823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:22:04.784546498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:22:04.793654767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:22:05.801857988Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=info ts=2026-05-10T19:22:07.877783084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:22:07.882192833Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:22:09.134200876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:22:09.135346746Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-10T19:22:10.233835747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:22:10.241144584Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:22:10.241167403Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:22:10.241193932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:22:10.248092353Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:22:11.467233265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:22:11.474431982Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:22:12.468774348Z caller=checkpoint.go:372 msg="writing series" size="661 kB"
level=warn ts=2026-05-10T19:22:14.13372356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:22:15.000758536Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33924"
level=debug ts=2026-05-10T19:22:15.801565627Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:22:15.801589546Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-10T19:22:19.042791155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:22:19.133661988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:22:19.133696139Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:22:19.133708732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:22:19.133736347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:22:19.140272574Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-05-10T19:22:19.142525779Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008241708s
level=info ts=2026-05-10T19:22:21.002597329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:22:21.013068904Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:22:24.134117641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:22:28.988103655Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:22:28.988160544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:22:28.997495711Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:22:29.133507286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:22:34.134365511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:22:39.133749191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:22:40.774354142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:22:40.785126232Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:22:40.78515543Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:22:40.785186265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:22:40.793582696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:22:44.133871359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:22:44.356909686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:22:44.366317215Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:22:44.703135439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:22:44.710745911Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:22:45.003551338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:51716"
level=debug ts=2026-05-10T19:22:45.209883118Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T19:22:47.791711964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47832"
ts=2026-05-10T19:22:49.046194136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:22:49.11676373Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:22:49.117826694Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T19:22:49.134193677Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:22:49.134204114Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:22:49.134225124Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:22:49.134236854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:22:49.134295116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:22:49.134364788Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-10T19:22:49.134374411Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:22:49.13439581Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.516µs success=true
level=debug ts=2026-05-10T19:22:49.134736996Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-10T19:22:50.462219384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:22:50.469925006Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:22:51.297759951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:22:51.307964877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:22:52.468869232Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T19:22:54.133618076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:22:55.801989633Z caller=checkpoint.go:372 msg="writing series" size="859 kB"
level=debug ts=2026-05-10T19:22:55.802012419Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-05-10T19:22:56.191951886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:22:56.199542694Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:22:59.133880623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:22:59.13492198Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-10T19:23:00.17608494Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:23:02.469032265Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-05-10T19:23:03.55343312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35584"
level=warn ts=2026-05-10T19:23:04.134242711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:05.802629641Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-10T19:23:05.802669335Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-10T19:23:07.811702517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:23:07.832434192Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:23:09.13340323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:09.13559249Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-10T19:23:09.157679184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:23:09.165407402Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:23:12.468488045Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:23:14.133846996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:15.801295855Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T19:23:17.794758866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43352"
ts=2026-05-10T19:23:19.049595024Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=warn ts=2026-05-10T19:23:19.133734271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:23:19.133743785Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:23:19.133770108Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:23:19.133781378Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:23:19.13546063Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-10T19:23:22.468215274Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-10T19:23:24.133815159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:25.801759979Z caller=checkpoint.go:372 msg="writing series" size="346 kB"
level=info ts=2026-05-10T19:23:26.416517587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:23:26.426593235Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:23:28.648224155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:23:29.134132225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:29.135325177Z caller=checkpoint.go:372 msg="writing series" size="157 kB"
level=debug ts=2026-05-10T19:23:32.468895026Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-10T19:23:33.556104897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39306"
level=warn ts=2026-05-10T19:23:34.13436433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:35.80213125Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-05-10T19:23:37.032762925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:23:37.042962106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:23:38.911942895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:23:38.919679971Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:23:39.133994892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:39.135285257Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-10T19:23:40.295238195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:23:40.297221404Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T19:23:42.468553756Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-05-10T19:23:42.468578582Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-05-10T19:23:43.321210842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:23:43.329603405Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:23:44.134188836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:45.801876201Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
ts=2026-05-10T19:23:49.052264224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:23:49.116976732Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:23:49.118073388Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:23:49.133488589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:23:49.133535864Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T19:23:49.133530429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:23:49.133546463Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:23:49.134670593Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=info ts=2026-05-10T19:23:50.050050444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:23:50.058236863Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:23:52.468987519Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:23:54.133392373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:55.801721583Z caller=checkpoint.go:372 msg="writing series" size="470 kB"
level=debug ts=2026-05-10T19:23:55.80174485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-10T19:23:57.872347933Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:23:59.134165629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:23:59.135351027Z caller=checkpoint.go:372 msg="writing series" size="166 kB"
level=debug ts=2026-05-10T19:24:02.468764961Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-05-10T19:24:04.1336824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:05.801768766Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-05-10T19:24:05.89681568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:24:05.90367596Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:24:09.133427889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:09.135759577Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=debug ts=2026-05-10T19:24:12.479206037Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-05-10T19:24:12.479228576Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=5
level=warn ts=2026-05-10T19:24:14.133706698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:24:15.010021639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53080"
level=debug ts=2026-05-10T19:24:15.804303483Z caller=checkpoint.go:372 msg="writing series" size="3.4 MB"
level=debug ts=2026-05-10T19:24:15.804329353Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=1
ts=2026-05-10T19:24:19.055197874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:24:19.134191555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:24:19.134220346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:24:19.134230496Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:24:19.134241144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:19.139218506Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-05-10T19:24:19.139231101Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.2 MB" series=1
level=debug ts=2026-05-10T19:24:21.930123281Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:24:22.469296272Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=info ts=2026-05-10T19:24:22.745755755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:24:22.754286172Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:24:24.134490978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:25.801745158Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:24:25.801768208Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-10T19:24:29.133629222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:29.13484773Z caller=checkpoint.go:372 msg="writing series" size="114 kB"
level=debug ts=2026-05-10T19:24:32.467998434Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-05-10T19:24:34.133512017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:35.802245454Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-05-10T19:24:36.076448696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:24:36.286391176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:24:36.294184936Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:24:36.846299889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:24:36.852593818Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:24:37.876485909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:24:37.883359907Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:24:39.133653346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:39.177940335Z caller=checkpoint.go:372 msg="writing series" size="48 MB"
level=debug ts=2026-05-10T19:24:39.177964291Z caller=checkpoint.go:384 msg="flushing series" totalSize="48 MB" series=4
level=info ts=2026-05-10T19:24:40.298170999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:24:40.306620704Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:24:42.468667136Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:24:44.133733627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:24:45.012932651Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34712"
level=debug ts=2026-05-10T19:24:45.802015765Z caller=checkpoint.go:372 msg="writing series" size="561 kB"
ts=2026-05-10T19:24:49.058177205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:24:49.116999376Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:24:49.117911419Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:24:49.134308848Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:24:49.134331951Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:24:49.134341066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:24:49.134346659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:49.13566983Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-10T19:24:52.468233625Z caller=checkpoint.go:372 msg="writing series" size="37 kB"
level=warn ts=2026-05-10T19:24:54.133821465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:54.409989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:24:55.80198756Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T19:24:59.134405996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:24:59.135698456Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:24:59.135714394Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T19:24:59.598343917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:24:59.603917977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:25:02.468239828Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
ts=2026-05-10T19:25:03.564084031Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:60232"
level=warn ts=2026-05-10T19:25:04.134334601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:25:05.171071077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:25:05.179920707Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:25:05.801606597Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T19:25:09.133833112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:09.135415414Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=debug ts=2026-05-10T19:25:12.468364825Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:25:14.133822629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:25:15.015712319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38698"
level=debug ts=2026-05-10T19:25:15.801475842Z caller=checkpoint.go:372 msg="writing series" size="640 B"
ts=2026-05-10T19:25:17.805468963Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59118"
ts=2026-05-10T19:25:19.061076291Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:25:19.134067888Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:25:19.134095166Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:25:19.13410865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:25:19.13412865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:19.135257401Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-05-10T19:25:20.611141091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:25:20.968107852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:25:20.979497268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:25:22.47267219Z caller=checkpoint.go:372 msg="writing series" size="5.2 MB"
level=debug ts=2026-05-10T19:25:22.47269435Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.9 MB" series=7
level=warn ts=2026-05-10T19:25:24.134193703Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:25.80162078Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:25:29.134117723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:29.135194177Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-10T19:25:32.46898219Z caller=checkpoint.go:372 msg="writing series" size="500 kB"
level=info ts=2026-05-10T19:25:33.056056394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:25:33.065758536Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:25:34.133619984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:34.805608835Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:25:35.80176256Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=info ts=2026-05-10T19:25:36.544588216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:25:36.552309877Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:25:38.48662843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:25:38.495707072Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:25:39.133755943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:39.134849035Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T19:25:42.468891418Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-05-10T19:25:44.133859103Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:45.802745517Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T19:25:45.802768279Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T19:25:47.260176759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:25:47.269591003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T19:25:47.808416491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:37972"
ts=2026-05-10T19:25:49.064408643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:25:49.117042031Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:25:49.117975564Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:25:49.134309127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:25:49.134327005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:49.134343191Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:25:49.134353306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:25:49.135457223Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-10T19:25:51.619754711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:25:52.468558461Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-05-10T19:25:54.134254399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:55.802090678Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T19:25:56.558993629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:25:56.565798355Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:25:59.134261792Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:25:59.13536139Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-10T19:26:01.774340092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:26:01.783064158Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:26:01.783090141Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:26:01.783129734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:26:01.791286121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:26:02.469207539Z caller=checkpoint.go:372 msg="writing series" size="694 kB"
level=warn ts=2026-05-10T19:26:04.134300738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:05.80133791Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:26:09.134286178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:09.136794503Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:26:09.136810207Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=debug ts=2026-05-10T19:26:12.468593907Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:26:14.133857239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:26:14.311769957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="19 MB" avg_uncomp="19 MB" full=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:26:14.323078262Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-10T19:26:15.019924329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35280"
level=debug ts=2026-05-10T19:26:15.802130192Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:26:16.890454892Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:26:17.659773349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:26:17.667706363Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:26:17.811005566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52148"
level=info ts=2026-05-10T19:26:18.526527029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="378 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:26:18.534492703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:26:19.067133509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:26:19.13374175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:26:19.133750093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:19.133770844Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:26:19.133781111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:26:19.134878458Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-10T19:26:21.635148906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:26:21.643442944Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:26:22.468023378Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-05-10T19:26:24.134131191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:25.801725505Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=warn ts=2026-05-10T19:26:29.133824636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:29.135095932Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-10T19:26:30.804312564Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:26:32.469051624Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-10T19:26:34.134196827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:35.802045502Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:26:35.802074702Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-10T19:26:39.133973866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:39.135354607Z caller=checkpoint.go:372 msg="writing series" size="763 kB"
level=debug ts=2026-05-10T19:26:41.533388002Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:26:42.46916915Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:26:42.469195131Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-10T19:26:44.134197847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:26:44.177297857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:26:44.186255658Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:26:45.802361447Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
ts=2026-05-10T19:26:49.070059981Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:26:49.117175593Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:26:49.11767243Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:26:49.134032583Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:26:49.134050128Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:26:49.134059052Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:26:49.134053297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:49.135393969Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-05-10T19:26:52.468370876Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-10T19:26:54.133375308Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:26:55.790718566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:26:55.799932716Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:26:55.799962155Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:26:55.799991835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:26:55.801259947Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=error ts=2026-05-10T19:26:55.804136871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:26:55.921699756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:26:55.92959663Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:26:59.134357249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:26:59.135452108Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T19:27:01.382848817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:27:01.392277073Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:27:02.469252549Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-10T19:27:02.46927518Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=6
ts=2026-05-10T19:27:03.573147974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49338"
level=warn ts=2026-05-10T19:27:04.134398629Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:27:05.357260066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:27:05.365532216Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:27:05.802076519Z caller=checkpoint.go:372 msg="writing series" size="291 kB"
level=debug ts=2026-05-10T19:27:08.260947747Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:27:09.133753268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:27:09.13499383Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=debug ts=2026-05-10T19:27:12.468520761Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=info ts=2026-05-10T19:27:13.652767825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:27:13.657502682Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:27:14.134262065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:27:15.802179475Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:27:15.802202761Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-10T19:27:16.038900461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:27:16.046535259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:27:17.81553Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43248"
ts=2026-05-10T19:27:19.072912071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:27:19.133317436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:27:19.133364354Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:27:19.133376918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:27:19.133407695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:27:19.142371365Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-05-10T19:27:19.144251316Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009717149s
level=warn ts=2026-05-10T19:27:24.134037801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:27:29.134100733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:27:29.54077106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:27:32.655234828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:27:34.134009306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:27:38.214185209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:27:38.223454949Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:27:39.133760201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:27:42.800125225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:27:42.804769212Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:27:42.805744452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:27:42.812601872Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:27:44.133656055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:27:49.075487699Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:27:49.117068476Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:27:49.117103683Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:27:49.133392112Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:27:49.13339847Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:27:49.133430637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:27:49.133455001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:27:49.133466928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:27:49.133542926Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-10T19:27:49.133550221Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:27:49.133568265Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.335µs success=true
level=debug ts=2026-05-10T19:27:49.13389896Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-10T19:27:49.33055025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="38 MB" avg_uncomp="19 MB" full=2 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:27:49.33895133Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:27:52.467659342Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T19:27:54.133595132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:27:55.802078602Z caller=checkpoint.go:372 msg="writing series" size="892 kB"
level=debug ts=2026-05-10T19:27:55.802101537Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-05-10T19:27:59.133669241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:27:59.134710722Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-10T19:28:00.40725408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:28:00.418313787Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:28:02.467891587Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-10T19:28:04.133469019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:05.720624737Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:28:05.801562791Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-10T19:28:05.801585046Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-05-10T19:28:06.934273242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:28:06.942580246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:28:07.2866137Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:28:07.288771259Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T19:28:09.133471324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:09.134605252Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=debug ts=2026-05-10T19:28:12.468015276Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:28:14.134122303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:28:15.029428431Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40562"
level=debug ts=2026-05-10T19:28:15.800972524Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
ts=2026-05-10T19:28:17.820458062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:50422"
ts=2026-05-10T19:28:19.079270805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:28:19.134077862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:28:19.134113682Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:28:19.134124391Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:28:19.134179531Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-10T19:28:19.134172927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:22.468163242Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-05-10T19:28:24.133965553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:25.801103584Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
level=info ts=2026-05-10T19:28:28.690054086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:28:28.697068363Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:28:29.133691909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:29.133930681Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=info ts=2026-05-10T19:28:29.748908331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:28:29.757163595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:28:32.467538318Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-10T19:28:34.134095365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:35.801368746Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-10T19:28:35.801392061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=info ts=2026-05-10T19:28:36.285899178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:28:36.293344438Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:28:37.024328323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:28:39.133713054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:39.133813918Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-10T19:28:42.467725955Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-10T19:28:44.134363101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:28:45.032746538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45936"
level=debug ts=2026-05-10T19:28:45.801376952Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-05-10T19:28:46.161976851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:28:47.197628669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:28:47.206386513Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:28:47.206406239Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:28:47.206430026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:28:47.213729941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-10T19:28:49.081910459Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:28:49.116971104Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:28:49.117325412Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:28:49.133617714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:28:49.133661407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:28:49.133684006Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:28:49.133692174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:28:49.13477959Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-10T19:28:52.468141749Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:28:54.133499336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:55.801439032Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
level=debug ts=2026-05-10T19:28:55.801461341Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T19:28:56.605752376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:28:56.615469124Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:28:56.615500394Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:28:56.615524463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:28:56.62265965Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:28:59.134144202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:28:59.134227402Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-10T19:29:02.467607301Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
level=warn ts=2026-05-10T19:29:04.134414492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:29:04.287757639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:29:04.295940568Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:29:05.800798539Z caller=checkpoint.go:372 msg="writing series" size="386 kB"
level=info ts=2026-05-10T19:29:06.379542839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:29:06.387195037Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:29:06.38722155Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:29:06.3872524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:29:06.393709438Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:29:07.633871371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:29:09.134299871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:09.134474839Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=info ts=2026-05-10T19:29:10.085893984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:29:10.090397102Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:29:12.454769546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:29:12.462135293Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:29:12.476229764Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-10T19:29:12.476247406Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=5
level=warn ts=2026-05-10T19:29:14.133974994Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:15.803319284Z caller=checkpoint.go:372 msg="writing series" size="3.5 MB"
level=debug ts=2026-05-10T19:29:15.803341553Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.5 MB" series=1
ts=2026-05-10T19:29:17.824674713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38256"
level=info ts=2026-05-10T19:29:18.495612549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:29:18.503037856Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:29:19.084765538Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=warn ts=2026-05-10T19:29:19.13416561Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:29:19.134205276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:29:19.134226661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:29:19.134238564Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:29:19.14150541Z caller=checkpoint.go:372 msg="writing series" size="7.9 MB"
level=debug ts=2026-05-10T19:29:19.141532728Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.9 MB" series=1
level=info ts=2026-05-10T19:29:20.801789386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="402 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:29:20.809403427Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:29:22.262058656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:29:22.271552173Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:29:22.27157857Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:29:22.271607571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:29:22.281795549Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:29:22.468945825Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T19:29:24.134251991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:25.800677791Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:29:25.80069995Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:29:27.714696013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:29:27.723236346Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:29:29.133460528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:29.134742595Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=debug ts=2026-05-10T19:29:30.225379608Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:29:32.405404818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:29:32.413081388Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:29:32.468221967Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-05-10T19:29:34.134116507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:35.8006103Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=debug ts=2026-05-10T19:29:36.171960846Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:29:39.133705648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:39.180278243Z caller=checkpoint.go:372 msg="writing series" size="52 MB"
level=debug ts=2026-05-10T19:29:39.180309335Z caller=checkpoint.go:384 msg="flushing series" totalSize="53 MB" series=4
level=debug ts=2026-05-10T19:29:42.467258455Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-10T19:29:43.977249356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:29:43.989991597Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:29:44.133493338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:29:45.037694559Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40300"
level=debug ts=2026-05-10T19:29:45.801128464Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
ts=2026-05-10T19:29:47.827080964Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46482"
ts=2026-05-10T19:29:49.088122739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:29:49.117224251Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:29:49.117845996Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:29:49.134176772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:29:49.134193039Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:29:49.134214598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:29:49.134223677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:29:49.134333996Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-05-10T19:29:49.895338159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:29:49.904410152Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:29:52.467581251Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-05-10T19:29:54.133760881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:29:54.394542718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:29:54.403024814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:29:55.801174308Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-05-10T19:29:56.28869511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:29:56.293797822Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:29:59.133967356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:29:59.134177921Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:29:59.134195344Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-10T19:30:02.467756768Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=debug ts=2026-05-10T19:30:02.605547809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:30:04.133938709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:05.801262535Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-10T19:30:07.548279918Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:30:09.134317155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:09.134668731Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:30:09.816418426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:30:09.825748472Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:30:12.467667675Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:30:14.134420869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:30:15.040488375Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55514"
level=debug ts=2026-05-10T19:30:15.801486294Z caller=checkpoint.go:372 msg="writing series" size="640 B"
ts=2026-05-10T19:30:19.090401468Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:30:19.133708384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:30:19.133713928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:19.133736605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:30:19.133747254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:30:19.133846106Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-10T19:30:19.231201044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:30:19.241410527Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:30:22.476369808Z caller=checkpoint.go:372 msg="writing series" size="8.6 MB"
level=debug ts=2026-05-10T19:30:22.476392156Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.2 MB" series=7
level=warn ts=2026-05-10T19:30:24.133386399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:25.800532735Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:30:29.134178152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:29.134197514Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-10T19:30:31.657287798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:30:32.467517029Z caller=checkpoint.go:372 msg="writing series" size="683 kB"
level=warn ts=2026-05-10T19:30:34.134202273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:35.801538098Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-05-10T19:30:38.345370573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:30:38.353772659Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:30:39.133360446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:39.134457872Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T19:30:42.467860372Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-10T19:30:42.467883808Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-10T19:30:44.133961687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:45.801226657Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=info ts=2026-05-10T19:30:48.003959274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:30:48.011785468Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
ts=2026-05-10T19:30:49.093938756Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:30:49.116979567Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:30:49.117747064Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T19:30:49.11898366Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T19:30:49.119023264Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976045 n=1
level=debug ts=2026-05-10T19:30:49.119063295Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976045 n=1
level=debug ts=2026-05-10T19:30:49.134073009Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-05-10T19:30:49.134089454Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:30:49.13409148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:49.134106165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:30:49.134121767Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:30:49.920375482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:30:50.389572365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:30:50.393847392Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:30:51.316524776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.1 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:30:51.32420991Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:30:52.467624417Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T19:30:54.133863808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:55.800967965Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-10T19:30:59.133856187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:30:59.13386087Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-10T19:31:02.468205084Z caller=checkpoint.go:372 msg="writing series" size="808 kB"
level=debug ts=2026-05-10T19:31:02.468228205Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=debug ts=2026-05-10T19:31:02.940337798Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:31:03.592437777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41402"
level=warn ts=2026-05-10T19:31:04.133606836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:05.800887649Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T19:31:07.554722767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="426 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:31:07.564802628Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:31:08.279560127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="6.5 MB" avg_comp="2.2 MB" total_uncomp="42 MB" avg_uncomp="14 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:31:08.287466272Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:31:09.13378079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:09.134581292Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T19:31:09.134596894Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=debug ts=2026-05-10T19:31:12.467597903Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:31:14.133611361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:15.801306625Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-10T19:31:19.09648525Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:31:19.133596451Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:31:19.133620756Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:31:19.133628845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:31:19.133676768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:19.134877715Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-10T19:31:22.467566427Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-10T19:31:24.133899705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:25.801222522Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-10T19:31:27.945316567Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:31:29.134327584Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-05-10T19:31:29.135737592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:32.468332609Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=warn ts=2026-05-10T19:31:34.134839251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:35.801615811Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:31:35.801660666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=debug ts=2026-05-10T19:31:36.541747876Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:31:36.544327099Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:31:36.694629937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:31:36.703372524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:31:39.136885494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:39.141001821Z caller=checkpoint.go:372 msg="writing series" size="767 kB"
level=debug ts=2026-05-10T19:31:42.468676692Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:31:42.468699713Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-10T19:31:42.954564681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:31:42.964264914Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:31:44.134110534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:31:45.046872272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33698"
level=debug ts=2026-05-10T19:31:45.80112835Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-05-10T19:31:46.256605627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:31:46.264971992Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T19:31:49.099296121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:31:49.116975152Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:31:49.117281233Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:31:49.133587108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:31:49.133613606Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:31:49.133623358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:31:49.13367481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:49.134945298Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-10T19:31:49.927920327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:31:49.934940072Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:31:52.467240823Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-05-10T19:31:53.457151487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:31:53.465044598Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:31:54.134191918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:55.800811036Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T19:31:59.13371272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:31:59.133813881Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-10T19:32:00.15182514Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:32:01.281264199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:32:01.290257481Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:32:02.469122071Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-05-10T19:32:02.469146645Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
ts=2026-05-10T19:32:03.598307584Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:36522"
level=warn ts=2026-05-10T19:32:04.135081297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:05.801029039Z caller=checkpoint.go:372 msg="writing series" size="341 kB"
level=debug ts=2026-05-10T19:32:09.134210797Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-10T19:32:09.134213731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:12.468471429Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=debug ts=2026-05-10T19:32:12.468497603Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-05-10T19:32:14.133371041Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:15.801407137Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
ts=2026-05-10T19:32:19.101815498Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=warn ts=2026-05-10T19:32:19.134251464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:19.134266648Z caller=checkpoint.go:384 msg="flushing series" totalSize="259 kB" series=1
level=info ts=2026-05-10T19:32:19.134372288Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:32:19.134385117Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:32:19.134395197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:32:19.143052419Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-05-10T19:32:19.145171464Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011446626s
level=debug ts=2026-05-10T19:32:22.529079879Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:32:24.1345716Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:32:26.834172166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:32:26.84386648Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:32:27.681223434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:32:27.689132707Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:32:28.470627712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:32:29.134347133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:32:30.56058484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:32:30.567105192Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:32:34.134098577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:32:38.129583495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:32:38.141837374Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:32:39.133933025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:32:44.133552388Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:32:49.104294021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:32:49.123723197Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:32:49.124580639Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:32:49.133722762Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:32:49.13374891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:32:49.133760447Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:32:49.133761928Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:32:49.13377407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:32:49.133911048Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-10T19:32:49.133918443Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:32:49.133935155Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.416µs success=true
level=debug ts=2026-05-10T19:32:49.134332826Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-10T19:32:52.22790947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:32:52.238438452Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:32:52.238460202Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:32:52.238500374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:32:52.245560996Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:32:52.468335718Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T19:32:53.094048334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:32:53.101963516Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:32:53.150498635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:32:54.134130241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:55.801863419Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:32:55.80188599Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-10T19:32:59.133576445Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:32:59.134780469Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-10T19:32:59.510611623Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:33:02.468015556Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=warn ts=2026-05-10T19:33:04.134016012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:33:05.610282467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:33:05.618397274Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:33:05.802816079Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-10T19:33:05.802838653Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-10T19:33:09.134259112Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=warn ts=2026-05-10T19:33:09.134270385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:12.468190512Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:33:14.134432809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:33:15.053548072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49614"
level=debug ts=2026-05-10T19:33:15.801126369Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=debug ts=2026-05-10T19:33:19.012322142Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:33:19.107603497Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:33:19.134104988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:33:19.13413074Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:33:19.134139162Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:33:19.134177504Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:19.134233773Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T19:33:22.298463257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:33:22.309276819Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:33:22.467713265Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=info ts=2026-05-10T19:33:23.275852668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:33:23.283814819Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:33:23.598758247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:33:23.603427219Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:33:23.603447164Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:33:23.603469413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:33:23.607286447Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:33:24.133906585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:25.801744676Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-05-10T19:33:29.133491666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:29.134817209Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-10T19:33:30.096050012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:33:30.104054015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:33:32.467800789Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-10T19:33:32.704476823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:33:32.711704614Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:33:34.133817896Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:35.800973195Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-10T19:33:35.801003219Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-05-10T19:33:39.13379119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:39.134965656Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:33:39.26721519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:33:39.274877246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:33:42.229979369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:33:42.235195733Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:33:42.467797961Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-10T19:33:44.134350869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:45.80116586Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-05-10T19:33:47.946314737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:33:48.638192869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:33:49.110577568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:33:49.11720235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:33:49.117606256Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:33:49.133967731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:33:49.133999707Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:33:49.134008097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:33:49.134026484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:49.135126101Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
level=debug ts=2026-05-10T19:33:52.468527719Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:33:54.13391901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:55.801845445Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=debug ts=2026-05-10T19:33:55.801869486Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-10T19:33:59.134392579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:33:59.134412576Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-05-10T19:34:02.467693043Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
ts=2026-05-10T19:34:03.607662615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35842"
level=warn ts=2026-05-10T19:34:04.134351157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:05.801422882Z caller=checkpoint.go:372 msg="writing series" size="471 kB"
level=info ts=2026-05-10T19:34:06.836101438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:34:06.848408477Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:34:09.133401911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:09.134619129Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-05-10T19:34:12.478420974Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-10T19:34:12.478446979Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=5
level=warn ts=2026-05-10T19:34:14.133841094Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:34:14.778761049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:34:14.787424021Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:34:15.396405416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="862 kB" avg_comp="862 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:34:15.396457256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-10T19:34:15.40136195Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:34:15.803233748Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-05-10T19:34:15.803256461Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.7 MB" series=1
ts=2026-05-10T19:34:17.84340904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39292"
ts=2026-05-10T19:34:19.113063514Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:34:19.134192847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:34:19.134220526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:34:19.134228826Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:34:19.134281172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:19.143876675Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-05-10T19:34:19.14389779Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=info ts=2026-05-10T19:34:21.321761495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:34:21.327735581Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:34:22.468220593Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T19:34:24.134336176Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:25.801939737Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:34:25.801966719Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:34:26.92960618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:34:26.936833473Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:34:27.525548269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:34:27.533861216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:34:29.134050766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:29.135151661Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-05-10T19:34:30.149104021Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:34:32.467924844Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-10T19:34:32.756929802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:34:32.761756023Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:34:34.133866322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:35.801115382Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-05-10T19:34:36.030850415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:34:36.039539134Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:34:39.134443914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:39.189930644Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-05-10T19:34:39.189984464Z caller=checkpoint.go:384 msg="flushing series" totalSize="57 MB" series=4
level=debug ts=2026-05-10T19:34:40.102427969Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:34:42.467702143Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:34:44.133669368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:45.802141688Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
ts=2026-05-10T19:34:47.846219232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:51552"
ts=2026-05-10T19:34:49.115266615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:34:49.117857605Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:34:49.119096991Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:34:49.133676768Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:34:49.133709463Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:34:49.133721084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:34:49.133763062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:49.134664492Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
level=debug ts=2026-05-10T19:34:52.468663447Z caller=checkpoint.go:372 msg="writing series" size="40 kB"
level=debug ts=2026-05-10T19:34:52.468696642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-05-10T19:34:54.014418769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:34:54.024137742Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:34:54.133950906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:55.801627846Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T19:34:59.133843253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:34:59.135164708Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:35:02.46810012Z caller=checkpoint.go:372 msg="writing series" size="1.5 kB"
level=warn ts=2026-05-10T19:35:04.133903454Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:05.078423861Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:35:05.080111873Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T19:35:05.801510848Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-10T19:35:07.202152554Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:35:09.133885034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:09.135363491Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:35:09.280233322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:35:09.285655955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:35:12.071691369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:35:12.076072037Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:35:12.467747051Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-10T19:35:12.983092777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:35:12.991079218Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:35:14.134284688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:35:15.062196529Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33722"
level=debug ts=2026-05-10T19:35:15.801806246Z caller=checkpoint.go:372 msg="writing series" size="640 B"
level=info ts=2026-05-10T19:35:16.460035089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:35:16.468823727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T19:35:17.848893633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47318"
ts=2026-05-10T19:35:19.117853121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:35:19.133348384Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:35:19.133374652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:35:19.133391175Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:35:19.133415577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:19.134546885Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-05-10T19:35:21.524106106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:35:21.532059636Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:35:22.477884495Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-10T19:35:22.477909362Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=9
level=warn ts=2026-05-10T19:35:24.134369397Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:25.80089762Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:35:29.134131419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:29.134167567Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-10T19:35:32.274984441Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:35:32.469261764Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=warn ts=2026-05-10T19:35:34.133582232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:35:35.766852062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:35:35.786628608Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:35:35.786675732Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:35:35.786702768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:35:35.793982693Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:35:35.801217479Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-05-10T19:35:39.134173951Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-05-10T19:35:39.134200823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:42.467919688Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-05-10T19:35:44.133693743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:35:45.068840408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:43244"
level=debug ts=2026-05-10T19:35:45.802174127Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T19:35:45.802203986Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-05-10T19:35:45.96405479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:35:45.971974679Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:35:48.985001113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:35:48.992515874Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:35:48.992539361Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:35:48.99256471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:35:48.999918733Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:35:49.116984256Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:35:49.11735776Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
ts=2026-05-10T19:35:49.120375996Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:35:49.133608393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:35:49.133609315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:49.133625677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:35:49.133648012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:35:49.134756708Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-05-10T19:35:50.241658798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:35:52.468291246Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T19:35:54.134462958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:35:55.40522244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:35:55.412273346Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:35:55.412295668Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:35:55.412319382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:35:55.419286863Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:35:55.801377053Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-10T19:35:55.968008029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:35:55.972087184Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:35:57.69309436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:35:57.697647993Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:35:59.133466411Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:35:59.134563366Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-05-10T19:36:01.901382344Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:36:02.468484506Z caller=checkpoint.go:372 msg="writing series" size="735 kB"
level=warn ts=2026-05-10T19:36:04.133900451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:05.801784698Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T19:36:06.981694159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:36:06.992178644Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:36:09.133367984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:09.136200565Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T19:36:09.136217075Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=7
level=info ts=2026-05-10T19:36:10.577704366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="450 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:36:10.587170741Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:36:11.072468391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:36:11.079799203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:36:12.468524001Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-05-10T19:36:13.369184919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:36:13.379540815Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:36:13.379569068Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:36:13.379593473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:36:13.3869014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:36:14.133470307Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:36:14.223695686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="9.4 MB" avg_comp="2.3 MB" total_uncomp="60 MB" avg_uncomp="15 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:36:14.231021976Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:36:15.801836982Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-10T19:36:17.853630723Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39624"
ts=2026-05-10T19:36:19.122954778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:36:19.134331233Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:36:19.134355583Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T19:36:19.134346212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:36:19.134364425Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:36:19.134415203Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-10T19:36:22.468265524Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-10T19:36:23.994596844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:36:24.00172963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:36:24.13366169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:25.801504779Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=info ts=2026-05-10T19:36:27.413089717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:36:27.420690134Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:36:29.134140567Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:29.134375202Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=debug ts=2026-05-10T19:36:32.468501587Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
ts=2026-05-10T19:36:33.61872764Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:52238"
level=warn ts=2026-05-10T19:36:34.133625559Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:35.801727322Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:36:35.801749054Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-10T19:36:35.873609963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:36:37.337280651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:36:37.344684333Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:36:39.133690368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:39.135500987Z caller=checkpoint.go:372 msg="writing series" size="811 kB"
level=debug ts=2026-05-10T19:36:42.468650512Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:36:42.468673617Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-10T19:36:44.134226492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:36:44.239500247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:36:44.24725584Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:36:45.682516627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:36:45.690224772Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:36:45.801755956Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-10T19:36:46.850907945Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T19:36:47.856127549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:39974"
level=info ts=2026-05-10T19:36:49.117204153Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:36:49.117233414Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
ts=2026-05-10T19:36:49.125537866Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:36:49.134132497Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:36:49.134136495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:49.134152945Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:36:49.134163835Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:36:49.134207881Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-10T19:36:49.776203727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:36:49.781505917Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:36:52.468589265Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-05-10T19:36:54.134588603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:55.80117533Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-05-10T19:36:59.13356118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:36:59.134687678Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T19:37:01.084117975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:37:01.09424822Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:37:02.470698531Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-10T19:37:02.47073313Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=6
level=warn ts=2026-05-10T19:37:04.134367187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:05.801260481Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-10T19:37:09.133971638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:09.135229167Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-10T19:37:09.973796722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:37:09.984977019Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:37:10.378104383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:37:12.468578047Z caller=checkpoint.go:372 msg="writing series" size="610 kB"
level=warn ts=2026-05-10T19:37:14.133503987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:15.801458671Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:37:15.801484092Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T19:37:18.246199924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:37:18.252931575Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-10T19:37:19.128502957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:37:19.134109641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:37:19.134148845Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T19:37:19.134158552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:37:19.134171508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:37:19.188205484Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-05-10T19:37:19.190834603Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.056728891s
level=info ts=2026-05-10T19:37:20.621104765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:37:20.630293194Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:37:24.133716006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:37:29.133432503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:37:29.4506982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:37:29.459668901Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:37:31.618288631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:37:31.623386315Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:37:33.626371558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:37:33.634877889Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:37:34.133889034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:37:39.134416159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:41.162665773Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:37:41.634953009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:37:41.646695228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:37:44.134326826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:44.234826857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:37:49.116976329Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:37:49.117415281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
ts=2026-05-10T19:37:49.131681713Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:37:49.13392795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:37:49.13394931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:37:49.133959968Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:37:49.133987496Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:37:49.134066367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:37:49.134150191Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-10T19:37:49.134161366Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:37:49.134183651Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.521µs success=true
level=debug ts=2026-05-10T19:37:49.134609725Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-10T19:37:52.468309701Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T19:37:54.134334444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:55.802434961Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:37:55.802464022Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-05-10T19:37:59.134145148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:37:59.135271031Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=info ts=2026-05-10T19:37:59.739859134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:37:59.751751062Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:38:01.278758511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:38:01.2840826Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:38:02.468591251Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-05-10T19:38:03.625412607Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:35562"
level=warn ts=2026-05-10T19:38:04.133769774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:05.80209209Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-05-10T19:38:05.802121412Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-05-10T19:38:09.134301526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:09.135296419Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-10T19:38:11.947449633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:38:11.958378962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:38:11.970411159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:38:11.977878315Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:38:12.468843157Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:38:14.133900301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:38:15.077407499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33550"
level=debug ts=2026-05-10T19:38:15.801589557Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=debug ts=2026-05-10T19:38:16.622195586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:38:16.897227688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:38:16.904774534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:38:19.133777523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:38:19.133816187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:38:19.13382676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:38:19.133871748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:38:19.134869366Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T19:38:19.135206213Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-05-10T19:38:22.468781689Z caller=checkpoint.go:372 msg="writing series" size="6.6 kB"
level=warn ts=2026-05-10T19:38:24.133562888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:24.831928539Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:38:25.072546777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:38:25.077405527Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:38:25.801816276Z caller=checkpoint.go:372 msg="writing series" size="502 kB"
level=warn ts=2026-05-10T19:38:29.133438065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:29.135677577Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-05-10T19:38:29.135698773Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-10T19:38:32.46866699Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-10T19:38:33.628725869Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38858"
level=warn ts=2026-05-10T19:38:34.133749558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:35.801706732Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=warn ts=2026-05-10T19:38:39.13345322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:39.134618655Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:38:40.408687467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:38:40.417680722Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:38:42.287969513Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:38:42.468716846Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-05-10T19:38:42.528104624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:38:42.534487189Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:38:44.134404857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:45.802144052Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-05-10T19:38:49.116978556Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:38:49.117421734Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:38:49.133700385Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:38:49.133738778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:38:49.133758864Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:38:49.133754134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:49.134904257Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:38:49.138149519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:38:52.401315657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:38:52.408995372Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:38:52.468482822Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:38:54.133412521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:55.80212492Z caller=checkpoint.go:372 msg="writing series" size="423 kB"
level=debug ts=2026-05-10T19:38:55.802147013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-10T19:38:59.133463346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:38:59.134753853Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-10T19:39:02.468304065Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=warn ts=2026-05-10T19:39:04.13379611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:05.80227446Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=info ts=2026-05-10T19:39:06.700553809Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:39:06.710682754Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:39:09.133713824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:09.135037708Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-05-10T19:39:09.183978225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:39:09.19254063Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:39:12.483100669Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-10T19:39:12.483124251Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=5
level=debug ts=2026-05-10T19:39:13.010278578Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:39:13.652140679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:39:13.661818752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:39:14.133879376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:15.803716707Z caller=checkpoint.go:372 msg="writing series" size="3.9 MB"
level=debug ts=2026-05-10T19:39:15.80373939Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=1
ts=2026-05-10T19:39:17.864588047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35484"
level=debug ts=2026-05-10T19:39:17.993041686Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:39:17.99504518Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:39:19.133448305Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:39:19.133475639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:39:19.133486134Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:39:19.133558523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:39:19.140708488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:39:19.148165972Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:39:19.148182332Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-05-10T19:39:22.46951103Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=info ts=2026-05-10T19:39:23.307166585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:39:23.312553815Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:39:24.134373451Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:25.802207324Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:39:25.802233268Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-10T19:39:29.133566658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:29.134774412Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-10T19:39:29.741292602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:39:29.749146398Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:39:32.468577075Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-10T19:39:34.134204731Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:35.801253779Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=info ts=2026-05-10T19:39:36.090891553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:39:36.098120626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:39:36.098142879Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:39:36.098177788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:39:36.104952221Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:39:39.133729275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:39.192075009Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-05-10T19:39:39.192110954Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=4
level=info ts=2026-05-10T19:39:39.520441947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:39:39.530812659Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:39:39.530849226Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:39:39.53088987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:39:39.539803476Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:39:42.469129653Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:39:44.133656285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:39:45.084550037Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50806"
level=debug ts=2026-05-10T19:39:45.782780368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:39:45.801932751Z caller=checkpoint.go:372 msg="writing series" size="550 kB"
level=info ts=2026-05-10T19:39:46.422129894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:39:46.441270395Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:39:48.598934053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:39:48.613852332Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:39:49.118519815Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:39:49.118572112Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:39:49.133414562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:39:49.133454134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:39:49.133465349Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:39:49.133465953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:49.134789099Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
ts=2026-05-10T19:39:49.143519563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:39:52.468804759Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-05-10T19:39:54.085258675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:39:54.090294064Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:39:54.134125309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:55.801382721Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-05-10T19:39:59.133416966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:39:59.134797395Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:39:59.134812669Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-10T19:40:01.43305235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:40:01.443942909Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:40:02.468421249Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-10T19:40:03.635749188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38946"
level=warn ts=2026-05-10T19:40:04.134089891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:05.801733196Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-10T19:40:07.839138359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:40:07.847355858Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:40:09.133400575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:09.135889038Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:40:10.936528634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:40:10.944790517Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:40:11.218330819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:40:11.226406564Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:40:12.468305652Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-10T19:40:13.504790076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:40:14.133895869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:40:15.087752006Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:44988"
level=debug ts=2026-05-10T19:40:15.801388191Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T19:40:17.810100477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:40:17.819810972Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:40:19.133318026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:40:19.133356853Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:40:19.133369306Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:40:19.133383303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:19.134475844Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:40:19.148706657Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:40:19.408057264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:40:19.408133689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:40:19.415574773Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=info ts=2026-05-10T19:40:20.607526292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:40:20.612685782Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:40:22.487170188Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:40:22.487193384Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=7
level=warn ts=2026-05-10T19:40:24.134265728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:25.801907224Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:40:29.133780481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:29.134794957Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-05-10T19:40:32.469105878Z caller=checkpoint.go:372 msg="writing series" size="702 kB"
level=warn ts=2026-05-10T19:40:34.133502128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:40:35.022192514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:40:35.032028674Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=debug ts=2026-05-10T19:40:35.802117312Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=info ts=2026-05-10T19:40:38.047752334Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:40:38.055425989Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:40:39.13374302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:39.13479621Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T19:40:42.468853539Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-05-10T19:40:42.468877865Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-10T19:40:44.133889255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:45.802607969Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T19:40:48.337155833Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:40:49.117015927Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:40:49.117018367Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:40:49.133353501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:40:49.133378297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:40:49.133389462Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:40:49.133402362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:49.134529117Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-10T19:40:49.151824479Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:40:49.943855904Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:40:52.467975583Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-10T19:40:54.133707907Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:40:54.323530408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:40:54.330804308Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:40:55.801394471Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-05-10T19:40:59.134174433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:40:59.135223032Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-10T19:41:00.353030406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:41:00.362323652Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:41:01.328118146Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.5 MB" total_uncomp="79 MB" avg_uncomp="16 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:41:01.336572236Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:41:02.468943135Z caller=checkpoint.go:372 msg="writing series" size="880 kB"
level=debug ts=2026-05-10T19:41:02.468967239Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-05-10T19:41:02.775311565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:41:02.779751848Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:41:02.834970118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:41:02.842965424Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:41:03.641138159Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48776"
level=warn ts=2026-05-10T19:41:04.134170614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:41:04.626439533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:41:04.633442749Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=debug ts=2026-05-10T19:41:05.801100124Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-05-10T19:41:09.133985433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:09.138371119Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-10T19:41:09.138392633Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=debug ts=2026-05-10T19:41:12.468673181Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-05-10T19:41:14.133549871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:41:15.092555749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33930"
level=debug ts=2026-05-10T19:41:15.801835462Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T19:41:17.470576133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:41:17.478933474Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-10T19:41:17.478956937Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:41:17.478981031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:41:17.483300914Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-05-10T19:41:17.873415126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:42922"
level=info ts=2026-05-10T19:41:19.133664156Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:41:19.133673355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:19.133692554Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:41:19.133701678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:41:19.134778225Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-10T19:41:19.154861562Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:41:22.468001878Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-05-10T19:41:24.133775279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:24.201010318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:41:25.802173892Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-05-10T19:41:28.176287702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:41:28.185045064Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:41:29.134117675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:29.13526019Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=debug ts=2026-05-10T19:41:32.468843494Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-05-10T19:41:34.13369866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:41:35.623745995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:41:35.631080165Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:41:35.801842199Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:41:35.801866697Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-05-10T19:41:38.773748707Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:41:39.133427633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:39.13491018Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=info ts=2026-05-10T19:41:39.96291227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:41:39.971127802Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:41:42.468619329Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:41:42.468673009Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=warn ts=2026-05-10T19:41:44.133691373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:41:45.485510331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:41:45.492024953Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=debug ts=2026-05-10T19:41:45.801995931Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-05-10T19:41:47.332465632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:41:47.340384261Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:41:47.876571707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33724"
level=info ts=2026-05-10T19:41:49.117159739Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:41:49.117543591Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:41:49.133817992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:41:49.133839648Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:41:49.133847637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:41:49.133859063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:49.135001545Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
ts=2026-05-10T19:41:49.158307856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:41:52.468793006Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=warn ts=2026-05-10T19:41:54.134291557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:55.801133883Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-10T19:41:55.916746903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:41:55.921733994Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:41:55.942900541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:41:55.94295395Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-10T19:41:55.949031677Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:41:58.932909163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:41:58.943323883Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:41:59.134189611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:41:59.135191811Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T19:41:59.966184213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:41:59.97445856Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:41:59.974484956Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:41:59.974514465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:41:59.982096172Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:42:02.470466036Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-10T19:42:02.470490114Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=warn ts=2026-05-10T19:42:04.134294769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:04.887761244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:42:04.893051987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:42:05.801787254Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-10T19:42:06.360384662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:42:06.36649005Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T19:42:09.133761221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:42:09.134946285Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-10T19:42:12.469307289Z caller=checkpoint.go:372 msg="writing series" size="571 kB"
level=warn ts=2026-05-10T19:42:14.13423215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:42:15.801748213Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:42:15.801772392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T19:42:18.129047087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:42:18.138468044Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:42:18.96466931Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:42:19.133630959Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:42:19.133658263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:42:19.133677052Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:42:19.133686032Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:42:19.14077456Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-05-10T19:42:19.142588364Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008239331s
ts=2026-05-10T19:42:19.161159815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=warn ts=2026-05-10T19:42:24.133552532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:27.546763933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:42:27.555182137Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=debug ts=2026-05-10T19:42:28.466157354Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:42:29.133403804Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:32.002536368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:42:32.009463488Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:42:33.332526504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:42:33.340307164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:42:33.340325808Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:42:33.340350191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:42:33.34782433Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:42:34.134222897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:34.837545093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:42:34.844884135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:42:34.844910831Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:42:34.84493853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:42:34.851927364Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:42:39.133686361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:41.072251616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:42:41.079726155Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:42:42.790554723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:42:42.798722571Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:42:42.798742734Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:42:42.798766375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:42:42.805763074Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:42:43.975192522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:42:43.979542533Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:42:44.134081706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:46.932998104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:42:46.940897389Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:42:49.117107388Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:42:49.117557365Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:42:49.133957354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:42:49.133985483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:42:49.13399361Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:42:49.13405954Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:42:49.134084925Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:49.13422863Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-10T19:42:49.134239485Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:42:49.134257799Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.965µs success=true
level=debug ts=2026-05-10T19:42:49.134595165Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T19:42:49.164525189Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:42:49.882203903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:42:49.890957272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:42:52.46812782Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-05-10T19:42:53.562223889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:42:53.563969563Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:42:53.926056354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:42:53.9308872Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:42:54.121498878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:42:54.130052326Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:42:54.134089973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:42:54.817415336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:42:54.825753005Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:42:55.802189739Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T19:42:55.802211431Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-10T19:42:59.13413902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:42:59.135261554Z caller=checkpoint.go:372 msg="writing series" size="2.6 kB"
level=debug ts=2026-05-10T19:42:59.583576976Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:43:02.46833178Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-05-10T19:43:04.133555933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:05.803165399Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-10T19:43:05.803187316Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=warn ts=2026-05-10T19:43:09.133552295Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:09.134743384Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-05-10T19:43:12.468087662Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=warn ts=2026-05-10T19:43:14.133373693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:43:15.51981339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:43:15.529900232Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:43:15.801700207Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=info ts=2026-05-10T19:43:17.24035705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:43:17.248383745Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:43:17.882913203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:44158"
level=info ts=2026-05-10T19:43:19.134223244Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:43:19.134251365Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:43:19.134260728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:43:19.134273702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:19.135429768Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:43:19.167536921Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T19:43:22.468817192Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-10T19:43:24.133531373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:43:24.801254437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:43:24.808488374Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=debug ts=2026-05-10T19:43:25.802266586Z caller=checkpoint.go:372 msg="writing series" size="398 kB"
level=info ts=2026-05-10T19:43:26.68114563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:43:26.68828573Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:43:27.025095551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:43:27.03490949Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:43:28.28896369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:43:28.294720566Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:43:28.91727073Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:43:29.134278754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:29.135607954Z caller=checkpoint.go:372 msg="writing series" size="286 kB"
level=info ts=2026-05-10T19:43:29.878563462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:43:29.883189323Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:43:32.468675962Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-10T19:43:32.624368078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T19:43:32.624426463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-10T19:43:32.628059462Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:43:32.924983738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:43:32.928426819Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:43:33.225241437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="47 kB" avg_uncomp="47 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:43:33.229088976Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:43:34.13354593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:35.801469942Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=debug ts=2026-05-10T19:43:35.801495107Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=9
level=warn ts=2026-05-10T19:43:39.133536606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:39.134712674Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:43:40.763548532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:43:40.77106588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:43:42.468058313Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T19:43:44.133699712Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:43:44.52316887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:43:44.526936926Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:43:45.416748289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:43:45.421570686Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:43:45.529971386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:43:45.53989842Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:43:45.801671064Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
ts=2026-05-10T19:43:47.886176803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38236"
level=info ts=2026-05-10T19:43:48.829577402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:43:48.833092339Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:43:49.116972396Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:43:49.117824031Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:43:49.13412401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:43:49.13414589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:43:49.13415621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:43:49.13417232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:49.135323931Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:43:49.170826398Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:43:52.468922583Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:43:54.133675063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:55.801510754Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=debug ts=2026-05-10T19:43:55.801534772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-10T19:43:59.133964544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:43:59.135110338Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-05-10T19:44:01.282573697Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:44:01.282589954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:44:01.288053506Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:44:01.583272683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="32 kB" avg_uncomp="32 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:44:01.587218459Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:44:01.883574934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:44:01.887776724Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:44:02.183125936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:44:02.187346594Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:44:02.468307878Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=warn ts=2026-05-10T19:44:04.134329867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:05.802123152Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-10T19:44:06.365780584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:44:06.373093369Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:44:07.403823899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:44:07.411673345Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:44:07.761564938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:44:07.765094768Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:44:09.134103888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:09.135514461Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=debug ts=2026-05-10T19:44:12.48052399Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-05-10T19:44:12.480547568Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=5
level=info ts=2026-05-10T19:44:12.593881992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:44:12.600123335Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:44:13.520568046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:44:13.528755262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:44:14.133922535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:44:14.177038014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:44:14.184099515Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:44:14.891460739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:44:14.894943148Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:44:15.296129652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:44:15.29945262Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T19:44:15.804613058Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-10T19:44:15.804653935Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=1
level=info ts=2026-05-10T19:44:16.084757672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:44:16.09285324Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=info ts=2026-05-10T19:44:16.240428801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:44:16.243916707Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:44:17.532311181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:44:17.536515849Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:44:17.901333781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:44:17.905071555Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:44:19.1337922Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:44:19.133830372Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:44:19.13383939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:44:19.133894435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:19.142790844Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:44:19.142810438Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-05-10T19:44:19.174186612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:44:19.489416543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:44:19.493265691Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:44:22.380376453Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:44:22.384400361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:44:22.391716994Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:44:22.468344953Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T19:44:24.134185721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:25.801511191Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:44:25.801534831Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:44:28.744061606Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:44:28.748369873Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:44:29.133852082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:29.134988387Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=debug ts=2026-05-10T19:44:32.468261899Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-10T19:44:34.134207426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:35.80173003Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-10T19:44:35.891071295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="165 kB" avg_uncomp="165 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:44:35.891120418Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-10T19:44:35.89497127Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T19:44:39.133732273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:39.200332011Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-10T19:44:39.200373629Z caller=checkpoint.go:384 msg="flushing series" totalSize="61 MB" series=4
level=info ts=2026-05-10T19:44:39.804719243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:44:39.818706457Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:44:42.468822268Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:44:44.133520135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:45.803252902Z caller=checkpoint.go:372 msg="writing series" size="630 kB"
level=info ts=2026-05-10T19:44:45.845904826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:44:45.856749066Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:44:47.841257064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:44:47.855370688Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:44:48.05503383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:44:48.065455471Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:44:48.108401386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:44:48.118834136Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T19:44:49.116988703Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:44:49.118984251Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:44:49.133853752Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:44:49.133902712Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:44:49.133913004Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:44:49.134020796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:49.134993583Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
ts=2026-05-10T19:44:49.176886773Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:44:51.1628497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:44:51.173293677Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:44:52.468896804Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=warn ts=2026-05-10T19:44:54.133403808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:44:54.603445032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:44:54.607059527Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:44:55.801269823Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-05-10T19:44:59.134241455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:44:59.135551052Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:44:59.135567364Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T19:44:59.15646111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:44:59.164734058Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:44:59.804974001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:44:59.812557658Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:45:00.471380394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:45:00.478002364Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:45:00.986379242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:45:00.994385504Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:45:02.468469544Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-10T19:45:03.659910355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57346"
level=warn ts=2026-05-10T19:45:04.133713306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:05.774236641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:45:05.801369278Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-10T19:45:06.277182804Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:45:09.133773491Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:09.135212499Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:45:09.142924113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:45:09.149704973Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=info ts=2026-05-10T19:45:12.291256153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:45:12.299771938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:45:12.407157921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:45:12.410777123Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:45:12.468903108Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:45:14.133900025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:15.80207899Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T19:45:15.956693312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:45:15.960929564Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:45:18.246208697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:45:18.251033397Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T19:45:19.133491525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:45:19.133490024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:45:19.133522051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:45:19.133531928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:45:19.134765692Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:45:19.180111647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:45:19.89998185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:45:19.908305499Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:45:22.486727409Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-10T19:45:22.486753262Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=7
level=info ts=2026-05-10T19:45:23.474227426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:45:23.478558231Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:45:24.134054698Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:25.801814633Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T19:45:29.133859878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:29.134944705Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=debug ts=2026-05-10T19:45:31.079331238Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:45:31.911493322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:45:31.915719612Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:45:32.468509817Z caller=checkpoint.go:372 msg="writing series" size="736 kB"
ts=2026-05-10T19:45:33.663357805Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:36064"
level=warn ts=2026-05-10T19:45:34.134229669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:45:35.600885058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:45:35.60613825Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:45:35.801889608Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-10T19:45:36.206086726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:45:36.21261007Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:45:37.053363522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:45:37.057407642Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:45:39.13339325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:39.135537223Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T19:45:39.909588018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:45:39.913825343Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:45:41.415235058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:45:41.423537384Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:45:41.423562955Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:45:41.423608522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:45:41.430547329Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:45:42.468272899Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-05-10T19:45:42.468297467Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-05-10T19:45:44.133987394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:45:45.108580113Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:49296"
level=debug ts=2026-05-10T19:45:45.802618548Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T19:45:49.116856013Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:45:49.117895833Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:45:49.119189886Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T19:45:49.119227421Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976046 n=1
level=debug ts=2026-05-10T19:45:49.119267083Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976046 n=1
level=info ts=2026-05-10T19:45:49.13418924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:45:49.134208678Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:45:49.134220224Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:45:49.134222977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:49.135302222Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-05-10T19:45:49.183427445Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:45:50.758904221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:45:50.767441745Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:45:52.423041734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:45:52.426789154Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:45:52.457851533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:45:52.464923229Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:45:52.468010068Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T19:45:52.887210894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:45:52.895834135Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=info ts=2026-05-10T19:45:53.872754649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:45:53.879484157Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:45:54.134390781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:45:54.508581381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:45:54.516327774Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:45:55.311499657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:45:55.801692334Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=info ts=2026-05-10T19:45:56.358473541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:45:56.362206852Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:45:59.133365297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:45:59.135480879Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-10T19:46:00.091500519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:46:00.100753207Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:46:00.413354147Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:46:02.469273436Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=debug ts=2026-05-10T19:46:02.469295746Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
ts=2026-05-10T19:46:03.666790436Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57092"
level=warn ts=2026-05-10T19:46:04.134353881Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:05.802186795Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T19:46:08.247690865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:46:08.25591933Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:46:09.13391877Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:09.136359641Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-10T19:46:09.13637458Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-10T19:46:09.640989419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:46:09.645299098Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:46:09.645325683Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:46:09.645347335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:46:09.649226277Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:46:10.690872384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:46:10.694194838Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:46:12.468771034Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-10T19:46:13.458693986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:46:13.462449946Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T19:46:14.134214405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:46:15.111512608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45040"
level=debug ts=2026-05-10T19:46:15.801802447Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T19:46:17.627469544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:46:17.631087772Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:46:18.315194014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:46:18.320048703Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:46:19.134017468Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:46:19.134055523Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:46:19.134064253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:46:19.134085518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:19.135173197Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-10T19:46:19.186487216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:46:22.468097313Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-05-10T19:46:23.93641634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:46:23.946804824Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:46:24.133665737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:25.801720758Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-10T19:46:26.663679495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:46:26.727977277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:46:26.731682743Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:46:26.825908216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:46:26.829528963Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:46:28.973934171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:46:28.979050046Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:46:29.13416675Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:29.135286515Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-05-10T19:46:32.468185869Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-05-10T19:46:33.86300131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:46:33.866503276Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:46:34.133675164Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:35.801940814Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:46:35.801962574Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-10T19:46:36.865183022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:46:36.872084367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:46:37.499317489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:46:37.50534895Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T19:46:39.134214379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:39.135926625Z caller=checkpoint.go:372 msg="writing series" size="799 kB"
level=info ts=2026-05-10T19:46:41.209374025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:46:41.213095634Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:46:42.373224923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:46:42.376930033Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:46:42.46848666Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:46:42.468506537Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-10T19:46:43.977542831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:46:43.985561746Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:46:44.13379585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:46:45.114312123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60488"
level=debug ts=2026-05-10T19:46:45.801679417Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-05-10T19:46:47.897358709Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40028"
level=debug ts=2026-05-10T19:46:48.824186076Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:46:48.825673291Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:46:49.116978292Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:46:49.117882905Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:46:49.134158868Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:46:49.134180322Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:46:49.134189148Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:46:49.134241065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:49.135420509Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
ts=2026-05-10T19:46:49.189693219Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:46:49.620911786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:46:49.628690426Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:46:49.663933919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ba4f5769e09a1120 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" full=1 labels="{stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:46:49.670678505Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ba4f5769e09a1120/19e13429da5%3A19e136727d6%3Abb4e4f03?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-kzbb\"}"
level=error ts=2026-05-10T19:46:49.670695589Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:46:49.67071601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:46:49.673401964Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:46:52.4683531Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=info ts=2026-05-10T19:46:52.484426808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:46:52.493127916Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:46:52.702910214Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:46:54.134313262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:55.801664456Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-05-10T19:46:57.053344112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:46:57.058979286Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T19:46:59.133673678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:46:59.134735238Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T19:47:00.131463909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:47:00.135619716Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:47:02.457602093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:47:02.462353457Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:47:02.469695565Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-05-10T19:47:02.46970992Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=6
level=info ts=2026-05-10T19:47:03.599531471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:47:03.603075743Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-10T19:47:03.672270794Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:55256"
level=warn ts=2026-05-10T19:47:04.133966989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:47:05.802369871Z caller=checkpoint.go:372 msg="writing series" size="301 kB"
level=warn ts=2026-05-10T19:47:09.133454534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:47:09.134707017Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-05-10T19:47:09.399176041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:47:09.402963963Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:47:09.732832915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:47:09.737700377Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T19:47:12.468470718Z caller=checkpoint.go:372 msg="writing series" size="571 kB"
level=warn ts=2026-05-10T19:47:14.134384209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:14.489246138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:47:14.496980254Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:47:15.116983921Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60930"
level=debug ts=2026-05-10T19:47:15.80157049Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:47:15.801594847Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
ts=2026-05-10T19:47:17.900268286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33674"
level=info ts=2026-05-10T19:47:19.134345065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:47:19.134374212Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:47:19.134382959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:47:19.134397039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:19.143726873Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-05-10T19:47:19.145697879Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011279513s
ts=2026-05-10T19:47:19.192845727Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:47:19.457981135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:47:19.464440588Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:47:20.979929783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:47:20.98392094Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:47:21.04314485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:47:21.046458087Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:47:21.370644839Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:47:24.134187079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:24.347046729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:47:24.35505854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:47:24.767535618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:47:24.771137307Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:47:26.527503838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:47:26.535793267Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:47:28.837776588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:47:28.843586619Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:47:29.133566714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T19:47:34.133751749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:36.402927472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:47:36.408472751Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:47:38.774227557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:47:38.778279042Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T19:47:39.133594743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:40.78549727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:47:40.794265417Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:47:40.832470725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:47:40.840782508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:47:43.264179709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:47:43.268136398Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T19:47:44.13406974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:47:47.474907223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-05-10T19:47:47.903054775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:35190"
level=info ts=2026-05-10T19:47:49.117117726Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:47:49.117131597Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:47:49.133393316Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:47:49.133411516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:47:49.133419242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:47:49.133425295Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:47:49.133486465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:47:49.133659476Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-10T19:47:49.133668566Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:47:49.133687172Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.333µs success=true
level=debug ts=2026-05-10T19:47:49.1340046Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T19:47:49.195880552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:47:49.250539143Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:47:50.543677372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:47:50.547309546Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:47:52.468216883Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T19:47:52.976755686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:47:52.981124417Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:47:54.133808511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:47:55.801630063Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T19:47:55.80167358Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-10T19:47:59.028219414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:47:59.033533038Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T19:47:59.1338596Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T19:47:59.133891Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:48:00.524503884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:48:00.529496722Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-10T19:48:00.529514843Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:48:00.529536794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:48:00.533351987Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:48:01.490685645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:48:01.496072534Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:48:01.496094183Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:48:01.49611639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:48:01.501290395Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:48:02.467860998Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=info ts=2026-05-10T19:48:02.631472439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:48:02.638502724Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:48:04.134110034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:05.803195825Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-05-10T19:48:05.803219003Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-05-10T19:48:05.83680018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:48:05.840526726Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:48:09.133383724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:09.134406937Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-10T19:48:09.868207666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:48:09.876444428Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:48:10.208597382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:48:10.212125223Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:48:12.467838162Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-10T19:48:14.001215353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:48:14.006212531Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:48:14.133739855Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:15.801581434Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T19:48:16.454325102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:48:16.458424738Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T19:48:17.800555401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:48:17.804312145Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-10T19:48:17.906077117Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:45336"
level=info ts=2026-05-10T19:48:18.121324033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:48:18.127048833Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:48:19.003359774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:48:19.133320025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:48:19.133348431Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:48:19.133366829Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:48:19.133358084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:19.134504774Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:48:19.198671276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:48:21.271975021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:48:21.280475139Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:48:22.467418236Z caller=checkpoint.go:372 msg="writing series" size="7.3 kB"
level=warn ts=2026-05-10T19:48:24.134234272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:25.801009122Z caller=checkpoint.go:372 msg="writing series" size="517 kB"
level=info ts=2026-05-10T19:48:29.117395481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:48:29.121371613Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T19:48:29.13345827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:29.13459975Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=debug ts=2026-05-10T19:48:32.46765755Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=warn ts=2026-05-10T19:48:34.134199997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:35.800620514Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T19:48:36.248027752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:48:36.252169284Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:48:37.675796586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:48:37.683428542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:48:38.277304175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:48:38.285105749Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:48:38.285129341Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:48:38.285160178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:48:38.292722227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:48:38.307735269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:48:38.311799013Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:48:38.689662082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:48:38.694524547Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:48:39.134248458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:39.134287844Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:48:40.237754011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:48:40.24283413Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:48:40.854526714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:48:40.858119856Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:48:42.468288445Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T19:48:44.134308572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:48:45.124151219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:55718"
level=debug ts=2026-05-10T19:48:45.801496999Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=debug ts=2026-05-10T19:48:45.801521775Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-05-10T19:48:46.818909501Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:48:48.475734531Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:48:49.117184857Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:48:49.117212033Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:48:49.133554923Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:48:49.133567727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:49.133589627Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:48:49.133605623Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:48:49.134702887Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:48:49.201232707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:48:52.46785138Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:48:54.13410263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:48:54.20421347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:48:54.212117639Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:48:54.476124796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:48:54.479794657Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:48:54.839988247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:48:54.847874387Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:48:54.847898778Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:48:54.847925505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:48:54.855107898Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:48:55.800691132Z caller=checkpoint.go:372 msg="writing series" size="419 kB"
level=info ts=2026-05-10T19:48:56.161711321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:48:56.169378649Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:48:57.765524356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:48:57.773553112Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:48:59.134142169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:48:59.13420946Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-05-10T19:48:59.134226285Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T19:49:00.503504176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:49:00.507887975Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:49:02.46826911Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
ts=2026-05-10T19:49:03.681363508Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39086"
level=info ts=2026-05-10T19:49:04.133349701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:49:04.133382726Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-10T19:49:04.136973711Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:49:04.508059048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:49:04.511603386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:49:05.801737392Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=info ts=2026-05-10T19:49:07.346530904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:49:07.354716165Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:49:07.354737492Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:49:07.354763929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:49:07.362169741Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:49:08.373008073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:49:08.380028547Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:49:09.134236515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:09.134454737Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=info ts=2026-05-10T19:49:12.274478859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:49:12.27849174Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:49:12.484263785Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-05-10T19:49:12.484285928Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=debug ts=2026-05-10T19:49:13.032196896Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:49:13.73131655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:49:13.735106523Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:49:14.133867173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:49:15.12759284Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:50308"
level=debug ts=2026-05-10T19:49:15.803112367Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-10T19:49:15.803138141Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=1
level=info ts=2026-05-10T19:49:19.134079491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:49:19.134107785Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:49:19.134119745Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:49:19.134115725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:19.146609896Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:49:19.146657078Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-05-10T19:49:19.204710038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:49:19.437907714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:49:19.44370027Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:49:20.024560957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:49:20.034351691Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:49:20.286216138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:49:20.294198319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:49:20.294222019Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:49:20.294248416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:49:20.301054198Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:49:21.747741538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:49:21.751370287Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:49:22.469281539Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T19:49:24.134359917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:49:24.83555857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:49:24.840471954Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:49:25.801237891Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:49:25.801263893Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:49:27.073678751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:49:27.081100042Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:49:29.133871968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:29.133944176Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=info ts=2026-05-10T19:49:30.449599538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:49:30.454092665Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:49:30.817204138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:49:30.825165483Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:49:32.467828216Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-10T19:49:33.684007288Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:53114"
level=warn ts=2026-05-10T19:49:34.134348357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:49:34.916284658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:49:34.920368359Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:49:35.207100851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:49:35.210729725Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:49:35.801435142Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=info ts=2026-05-10T19:49:35.994993547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:49:36.001563098Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:49:36.474652799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:49:36.48114709Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:49:38.214648565Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:49:39.133657753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:39.196680514Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-05-10T19:49:39.196715453Z caller=checkpoint.go:384 msg="flushing series" totalSize="61 MB" series=4
level=info ts=2026-05-10T19:49:40.648098989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:49:40.656693458Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:49:42.467753666Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=warn ts=2026-05-10T19:49:44.134080815Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:45.802143145Z caller=checkpoint.go:372 msg="writing series" size="641 kB"
level=info ts=2026-05-10T19:49:45.829388137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:49:45.846820026Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:49:46.28142457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:49:46.289213197Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:49:46.328556791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:49:46.332235171Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:49:46.332261183Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:49:46.332286759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:49:46.335235477Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:49:47.323673892Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:49:49.116890904Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:49:49.117948846Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T19:49:49.13334016Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:49:49.133370358Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:49:49.13337905Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:49:49.133412007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:49.134596136Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
ts=2026-05-10T19:49:49.207982169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T19:49:52.467779157Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=info ts=2026-05-10T19:49:53.038361366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:49:53.042580906Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T19:49:54.134233382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:55.801486878Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-10T19:49:58.304756753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:49:58.309033554Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:49:59.134059651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:49:59.134246688Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:49:59.134260976Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T19:49:59.959140228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:49:59.963075602Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:49:59.963097383Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:49:59.963120975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:49:59.96630558Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:50:01.41543322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:50:01.41992393Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:50:02.467481462Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
ts=2026-05-10T19:50:03.687118303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47970"
level=warn ts=2026-05-10T19:50:04.133932905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:05.80081406Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-10T19:50:08.919551719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:50:08.928276696Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:50:09.134165058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:09.134517362Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:50:09.511609883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:50:09.515143595Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:50:12.467508436Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:50:14.134004763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:50:15.697896057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:50:15.702335528Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:50:15.800731758Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T19:50:17.058224041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:50:17.06315539Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:50:18.297682273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:50:18.301194446Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:50:18.57314571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:50:18.576536152Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-10T19:50:18.576555572Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:50:18.576575173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:50:18.579607844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:50:18.631840118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:50:18.637056394Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:50:19.13367573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:50:19.13371229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:50:19.133733328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:50:19.133760242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:19.134950726Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:50:19.210202668Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:50:22.377417107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:50:22.487969577Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-05-10T19:50:22.487999274Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=7
level=warn ts=2026-05-10T19:50:24.13425443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:50:24.572097169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.5 MB" total_uncomp="98 MB" avg_uncomp="16 MB" full=5 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:50:24.580235349Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:50:25.800568876Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-10T19:50:25.936207221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="475 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:50:25.944225016Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:50:26.849480837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:50:26.858764288Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:50:27.357228042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:50:27.360761607Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:50:29.134208799Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T19:50:29.13428171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:50:30.756841377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:50:30.761949762Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:50:32.468670216Z caller=checkpoint.go:372 msg="writing series" size="562 kB"
level=info ts=2026-05-10T19:50:33.438223236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:50:33.442526359Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:50:33.442547554Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:50:33.442572782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:50:33.445424447Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:50:33.668161011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:50:33.674591484Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-10T19:50:33.690057961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49246"
level=warn ts=2026-05-10T19:50:34.134324901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:50:35.371559198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:50:35.374958654Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:50:35.374981324Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:50:35.375003566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:50:35.377818423Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:50:35.633585736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:50:35.640941051Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:50:35.801612947Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-05-10T19:50:36.572178968Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:50:39.133995749Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:39.134075287Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-10T19:50:42.468416484Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=debug ts=2026-05-10T19:50:42.468442112Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-05-10T19:50:44.134016573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:45.802004133Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
ts=2026-05-10T19:50:47.915966437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49402"
level=info ts=2026-05-10T19:50:49.117963978Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:50:49.119082867Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:50:49.134275028Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:50:49.134293668Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:50:49.134304141Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:50:49.134356757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:49.134388128Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-10T19:50:49.213176237Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:50:50.984921463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:50:50.991817986Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:50:50.991844568Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:50:50.991871101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:50:50.995230624Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:50:52.119600143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:50:52.123554722Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:50:52.388277839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:50:52.392375981Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:50:52.39240029Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:50:52.392426196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:50:52.395731587Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:50:52.467980215Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-05-10T19:50:53.259462764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:50:53.268109578Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:50:54.016365986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:50:54.134148948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:50:54.200263472Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:50:54.204524139Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:50:55.801518285Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=info ts=2026-05-10T19:50:56.601754776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:50:56.605533876Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:50:59.133866887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:50:59.133899726Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-10T19:51:01.631869111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:51:01.635773506Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:51:01.635801296Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:51:01.635826403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:51:01.638924268Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:51:02.468400359Z caller=checkpoint.go:372 msg="writing series" size="939 kB"
level=debug ts=2026-05-10T19:51:02.468426079Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=warn ts=2026-05-10T19:51:04.133755085Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:51:04.574072031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:51:04.579440207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:51:05.529519937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:51:05.537545926Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:51:05.801196602Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-05-10T19:51:06.68838175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="499 MB" avg_uncomp="24 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:51:06.697263447Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:51:07.010301023Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:51:07.013833022Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:51:07.320072427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:51:07.325171048Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:51:07.508569674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:51:07.512120246Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T19:51:09.133998472Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:09.137127446Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-10T19:51:09.137160579Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=info ts=2026-05-10T19:51:10.356298474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:51:10.359824394Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:51:12.468252118Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-10T19:51:13.306237683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:51:13.309837438Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T19:51:14.133790575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:51:15.135430252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:42610"
level=debug ts=2026-05-10T19:51:15.801702428Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T19:51:17.318070796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:51:17.321850997Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:51:17.513355418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:51:17.51718827Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:51:18.819858405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:51:18.826072326Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:51:19.133739898Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:51:19.133766789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:19.133781798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:51:19.133801131Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:51:19.134311028Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-10T19:51:19.215857603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:51:20.586339897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:51:20.589749941Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:51:22.320832515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:51:22.327903183Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:51:22.468289344Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=debug ts=2026-05-10T19:51:22.785341155Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:51:22.804967587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:51:22.806491997Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T19:51:24.133613015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:25.801421397Z caller=checkpoint.go:372 msg="writing series" size="280 kB"
level=info ts=2026-05-10T19:51:26.817914598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:51:26.821534443Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T19:51:27.425233675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:51:27.432323619Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:51:29.133571963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:29.134728259Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=info ts=2026-05-10T19:51:29.40252674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:51:29.406042208Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:51:32.468366275Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-10T19:51:33.91391767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:51:33.922344723Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:51:34.134054242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:35.801150682Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:51:35.80117809Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=8
level=warn ts=2026-05-10T19:51:39.134322857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:39.135056142Z caller=checkpoint.go:372 msg="writing series" size="844 kB"
level=debug ts=2026-05-10T19:51:42.467911826Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T19:51:42.467934963Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-10T19:51:42.483538747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:51:42.489607509Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:51:44.133826276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:51:45.137919183Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:46332"
level=debug ts=2026-05-10T19:51:45.801356446Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-10T19:51:47.504240723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:51:47.509611896Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T19:51:47.639009113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:51:47.642629311Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-10T19:51:47.921276634Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49176"
level=debug ts=2026-05-10T19:51:48.016907341Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:51:48.225860872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:51:48.230740499Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:51:48.721164537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="499 MB" avg_uncomp="24 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:51:48.729719618Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:51:48.729740711Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:51:48.729781325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="499 MB" avg_uncomp="24 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:51:48.7375158Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:51:49.11705838Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:51:49.117468152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:51:49.133735733Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:51:49.13376425Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:51:49.133774833Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:51:49.133804601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:49.133959992Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
ts=2026-05-10T19:51:49.21862907Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:51:50.114885947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:51:50.123147769Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:51:51.027777335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:51:51.031278142Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:51:52.467882683Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=warn ts=2026-05-10T19:51:54.134154061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:55.801442378Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T19:51:56.729262812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:51:56.734693038Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T19:51:59.133712914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:51:59.134841444Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-05-10T19:51:59.83399236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:52:00.356577033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:52:00.360563315Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:52:00.450793922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="499 MB" avg_uncomp="24 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:52:00.459452099Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:52:02.469600992Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-10T19:52:02.46962489Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=6
level=info ts=2026-05-10T19:52:02.68910793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:52:02.694940726Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-05-10T19:52:03.697059365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:54912"
level=warn ts=2026-05-10T19:52:04.133876271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:52:05.801507519Z caller=checkpoint.go:372 msg="writing series" size="362 kB"
level=warn ts=2026-05-10T19:52:09.133684826Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:52:09.13499954Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-05-10T19:52:10.589962942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:52:10.594126916Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:52:12.468473504Z caller=checkpoint.go:372 msg="writing series" size="571 kB"
level=debug ts=2026-05-10T19:52:12.468515489Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T19:52:13.211739513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:52:13.216271276Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T19:52:14.133551614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:52:15.141063757Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:35662"
level=info ts=2026-05-10T19:52:15.22395923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:52:15.232502314Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:52:15.801302374Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-10T19:52:15.948753142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:52:15.952658779Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:52:19.133315739Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:52:19.133345211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:52:19.133357521Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:52:19.133393905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:52:19.134416006Z caller=checkpoint.go:384 msg="flushing series" totalSize="259 kB" series=1
level=info ts=2026-05-10T19:52:19.143413619Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-05-10T19:52:19.144990807Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011185855s
ts=2026-05-10T19:52:19.221950639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:52:20.101025824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:52:20.108936391Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:52:20.122348078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:52:24.133942871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:52:24.820203881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:52:24.826591006Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:52:29.133672263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:52:29.705544803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:52:29.709737352Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:52:30.395575954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="93 MB" avg_uncomp="23 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:52:30.403749916Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:52:32.510684794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:52:32.519258432Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:52:34.133572123Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:52:37.218163419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:52:37.222155533Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:52:38.858005098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=21 total_comp="61 MB" avg_comp="2.9 MB" total_uncomp="499 MB" avg_uncomp="24 MB" full=21 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:52:38.865419114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 21, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:52:39.133911463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:52:40.683259965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:52:40.687796483Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:52:40.687829598Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:52:40.687857474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:52:40.691721537Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:52:41.73702027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:52:41.742582142Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:52:44.134464739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:52:45.144341618Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47034"
level=debug ts=2026-05-10T19:52:48.176274289Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:52:49.11690699Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:52:49.117969384Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T19:52:49.133291939Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T19:52:49.133497044Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-10T19:52:49.133508051Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:52:49.133531505Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=17.299µs success=true
level=info ts=2026-05-10T19:52:49.13355773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:52:49.133582207Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:52:49.13359429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:52:49.133607407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:52:49.13390848Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T19:52:49.225031296Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:52:50.02373952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:52:50.027379377Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:52:50.0434492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:52:50.046857507Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:52:52.058587479Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:52:52.467777609Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T19:52:52.888760171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:52:52.893355398Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:52:54.13407111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:52:54.965556255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:52:54.969651215Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:52:55.245497395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:52:55.249014578Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:52:55.55384505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:52:55.557192122Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:52:55.801221239Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-10T19:52:55.801243969Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=info ts=2026-05-10T19:52:58.754834263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:52:58.762103042Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:52:59.134234284Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T19:52:59.134254475Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:02.467244461Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=info ts=2026-05-10T19:53:02.614680658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:53:02.621739301Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T19:53:04.133792492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:05.801983857Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-10T19:53:05.802012623Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.0 MB" series=3
level=warn ts=2026-05-10T19:53:09.134348016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:09.134368276Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-10T19:53:10.910250404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:53:10.917456277Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:53:11.549700908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:53:11.553486017Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:53:12.46759607Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-10T19:53:13.349167555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:53:13.353722296Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:53:14.133817935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:53:14.264099309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:53:14.267628175Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:53:15.801052491Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T19:53:17.617279953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:53:17.625378292Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:53:19.133664236Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:53:19.133695134Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:53:19.133704445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:53:19.133765696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:19.134011093Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:53:19.228397156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:53:22.467659093Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-05-10T19:53:22.563109136Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:53:24.134187061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:53:24.205365844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:53:24.209332583Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:53:24.304457778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:53:24.313326867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:53:25.801658916Z caller=checkpoint.go:372 msg="writing series" size="456 kB"
level=warn ts=2026-05-10T19:53:29.13429497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:29.134340405Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-05-10T19:53:29.406829658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:53:29.415249018Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:53:32.456050356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:53:32.459724475Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T19:53:32.467767103Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-10T19:53:33.7042366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:41358"
level=info ts=2026-05-10T19:53:33.799063507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:53:33.804130269Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T19:53:34.133409578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:35.800981039Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T19:53:35.837187968Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:53:35.840544193Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:53:35.840566216Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:53:35.840592735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:53:35.84345235Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:53:37.343497619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:53:39.134375361Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:39.13441117Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-05-10T19:53:42.467689526Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T19:53:44.133424641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:53:44.987173431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:53:44.991048667Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
ts=2026-05-10T19:53:45.148351489Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:59206"
level=info ts=2026-05-10T19:53:45.517083651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:53:45.520775832Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:53:45.66503863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:53:45.668515619Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:53:45.801041604Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=debug ts=2026-05-10T19:53:45.801066515Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=info ts=2026-05-10T19:53:48.618852234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:53:48.627297222Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:53:49.117001241Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:53:49.117246797Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:53:49.133606848Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T19:53:49.133661993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:49.133680462Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:53:49.133693096Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:53:49.13380215Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:53:49.231388539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:53:49.629228483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:53:49.632948325Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:53:49.924906571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:53:49.928831224Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:53:52.468149338Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:53:54.134261681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:53:54.220545713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:53:54.224562925Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T19:53:55.756943427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:53:55.761318783Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T19:53:55.801595208Z caller=checkpoint.go:372 msg="writing series" size="518 kB"
level=debug ts=2026-05-10T19:53:55.801615836Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-05-10T19:53:57.906914064Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T19:53:59.134067024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:53:59.134214351Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-05-10T19:54:00.471662176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:54:00.478014353Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:54:02.468185943Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-05-10T19:54:02.624755231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:54:02.628228432Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-10T19:54:03.707183837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59076"
level=warn ts=2026-05-10T19:54:04.134128016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:05.800761196Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-05-10T19:54:06.32631891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:54:06.33608605Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:54:07.527022288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:54:07.53453639Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:54:09.133543017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:09.134911095Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-05-10T19:54:09.279988919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:54:09.283602893Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:54:12.482224679Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:54:12.482249071Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=5
level=warn ts=2026-05-10T19:54:14.134093227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:54:14.420606432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:54:14.429776704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:54:15.151124857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47570"
level=debug ts=2026-05-10T19:54:15.517728694Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:54:15.805710687Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-05-10T19:54:15.805738812Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.1 MB" series=1
ts=2026-05-10T19:54:17.93041532Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47900"
level=info ts=2026-05-10T19:54:19.133322356Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:54:19.133355232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:54:19.133364553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:54:19.133406157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:19.148729675Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:54:19.148759286Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-05-10T19:54:19.234195643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:54:20.516539027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:54:20.525695932Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:54:20.541747455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:54:20.545307898Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:54:22.468134146Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=info ts=2026-05-10T19:54:22.623164904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:54:22.626764655Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:54:24.133420941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:25.515703184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:54:25.800673712Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:54:25.800698517Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:54:27.363404051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T19:54:27.369034569Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-10T19:54:27.369056129Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:54:27.369077521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="110 B" avg_comp="110 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" idle=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:54:27.372150648Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-10T19:54:29.134123668Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:29.13415577Z caller=checkpoint.go:372 msg="writing series" size="165 kB"
level=info ts=2026-05-10T19:54:29.51033693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:54:29.513887025Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T19:54:32.468001347Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-10T19:54:34.133906888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:35.80111145Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=warn ts=2026-05-10T19:54:39.133511701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:39.205466322Z caller=checkpoint.go:372 msg="writing series" size="62 MB"
level=debug ts=2026-05-10T19:54:39.205504268Z caller=checkpoint.go:384 msg="flushing series" totalSize="62 MB" series=4
level=info ts=2026-05-10T19:54:40.339790975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:54:40.352580357Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:54:41.044349172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:54:41.053440128Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:54:41.436802161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:54:41.444868427Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:54:42.467688868Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-05-10T19:54:43.262938919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:54:43.274162541Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T19:54:44.134456201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:54:44.389951896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:54:44.408331667Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:54:44.70742119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:54:44.718670889Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:54:45.80147289Z caller=checkpoint.go:372 msg="writing series" size="493 kB"
level=info ts=2026-05-10T19:54:45.893684772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:54:45.90447724Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T19:54:49.117016901Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:54:49.119140792Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T19:54:49.133769297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:54:49.133948543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:54:49.133967665Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:54:49.133978737Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:54:49.133982417Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
ts=2026-05-10T19:54:49.237416807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:54:49.660612855Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:54:52.467820978Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
level=debug ts=2026-05-10T19:54:53.168064477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:54:53.170095921Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T19:54:53.344695355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:54:53.349182588Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:54:54.133751211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:54:54.84618552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:54:54.849803756Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:54:55.609258512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:54:55.61679518Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:54:55.801313485Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-10T19:54:57.547350487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:54:57.554335541Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:54:57.55436169Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:54:57.554388439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:54:57.560729926Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T19:54:59.133519489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:54:59.134875558Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-10T19:55:01.888686672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:55:01.8970162Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:55:01.897042794Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:55:01.897077324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:55:01.905192274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:55:02.467666413Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-10T19:55:02.653210759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:55:02.66301446Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T19:55:04.134262625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:05.801399138Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=debug ts=2026-05-10T19:55:05.80142138Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-05-10T19:55:07.000709056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:55:07.004772374Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:55:09.133412913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:09.135089694Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T19:55:10.298469697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:55:10.30482259Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:55:11.200714461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:55:11.205745014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T19:55:11.410447474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:55:11.414285538Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:55:12.189483501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:55:12.197952417Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:55:12.467806129Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T19:55:14.134048436Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:55:15.15598418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34612"
level=info ts=2026-05-10T19:55:15.27951136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:55:15.287952579Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T19:55:15.486669797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:55:15.800661265Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T19:55:19.134082745Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:55:19.134113943Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:55:19.13412275Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:55:19.134177645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:19.134191538Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T19:55:19.244791854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:55:20.43305744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:55:20.437304604Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:55:22.486335861Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-05-10T19:55:22.486363298Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=5
level=warn ts=2026-05-10T19:55:24.134331797Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:25.800549367Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-10T19:55:27.167246161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:55:27.172003851Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:55:28.289106336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:55:28.290726695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:55:28.293263841Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:55:28.298890545Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:55:28.298908965Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:55:28.298938134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:55:28.306797228Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:55:29.134060394Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T19:55:29.134084229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:55:30.99586801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:55:30.999376276Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:55:32.46799719Z caller=checkpoint.go:372 msg="writing series" size="563 kB"
level=info ts=2026-05-10T19:55:33.485572802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:55:33.48932309Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T19:55:34.134361011Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:35.801711752Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=info ts=2026-05-10T19:55:38.942953593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:55:38.947074023Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T19:55:39.134164433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:39.134208588Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T19:55:42.403206041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:55:42.410302191Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:55:42.468215109Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
level=debug ts=2026-05-10T19:55:42.468235992Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T19:55:42.776049962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:55:42.779706814Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:55:43.467399647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:55:43.47561305Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T19:55:43.729525546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:55:43.737906975Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:55:43.897808435Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:55:44.133848968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:55:44.865848173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:55:44.873486172Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:55:44.873515124Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:55:44.873542964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:55:44.880485946Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T19:55:45.158679318Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37024"
level=debug ts=2026-05-10T19:55:45.711110984Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:55:45.801736102Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=info ts=2026-05-10T19:55:48.842123099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:55:48.846515382Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:55:49.117113374Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:55:49.117617974Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:55:49.133991277Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:55:49.134019609Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:55:49.134028649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:55:49.134077146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:49.134102804Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-10T19:55:49.247820211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:55:50.443468185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:55:50.447625623Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T19:55:52.467950501Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T19:55:54.133692998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:55:55.055193592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:55:55.063215181Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:55:55.241949517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:55:55.24964757Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:55:55.801197088Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=warn ts=2026-05-10T19:55:59.133870638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:55:59.133933821Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-10T19:56:02.471747114Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-05-10T19:56:02.471778874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=6
ts=2026-05-10T19:56:03.717761327Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:45514"
level=warn ts=2026-05-10T19:56:04.134074758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:56:04.665854831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:56:04.670206347Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T19:56:05.801501859Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T19:56:05.860784569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:56:05.869255427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:56:05.869279443Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:56:05.869310178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:56:05.877134668Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:56:09.134149105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:09.137290107Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T19:56:09.137311293Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-10T19:56:10.08076604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="630 B" avg_comp="630 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:56:10.084530112Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T19:56:10.38654099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:56:10.39019378Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:56:11.608134706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:56:11.613105431Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T19:56:12.467502687Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-10T19:56:13.603952986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:56:13.607776889Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:56:14.132411207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:56:14.133545317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-10T19:56:14.136260925Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:56:14.136285447Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:56:14.136309784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:56:14.139437818Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:56:14.231869934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:56:15.162186932Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45206"
level=debug ts=2026-05-10T19:56:15.801668302Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T19:56:16.834115697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:56:16.841929754Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T19:56:17.938319781Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54762"
level=info ts=2026-05-10T19:56:19.133755438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:56:19.133794234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:56:19.133807545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:56:19.133805789Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=warn ts=2026-05-10T19:56:19.133848492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:56:19.250404561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T19:56:22.193334236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:56:22.202160981Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:56:22.468164803Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=info ts=2026-05-10T19:56:23.385781004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:56:23.38940326Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:56:24.134094167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:56:24.587693282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:56:24.59596188Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T19:56:24.620153452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:56:24.623563037Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T19:56:25.801177139Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=info ts=2026-05-10T19:56:29.124609101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:56:29.128208513Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T19:56:29.133857783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:29.134189922Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=info ts=2026-05-10T19:56:30.292248318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:56:30.301304246Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:56:32.467739342Z caller=checkpoint.go:372 msg="writing series" size="146 kB"
level=debug ts=2026-05-10T19:56:32.467765559Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-10T19:56:32.676179742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:56:32.682473975Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:56:33.676272014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:56:33.68076145Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:56:34.134616191Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:35.801619068Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T19:56:35.93710709Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T19:56:38.728513526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:56:38.732073417Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:56:38.986764882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:56:38.990392232Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:56:38.990419238Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:56:38.990443396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:56:38.993619989Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:56:39.13397618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:39.134567919Z caller=checkpoint.go:372 msg="writing series" size="882 kB"
level=debug ts=2026-05-10T19:56:39.134584941Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=debug ts=2026-05-10T19:56:42.468054883Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=warn ts=2026-05-10T19:56:44.1341985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:45.800899185Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=debug ts=2026-05-10T19:56:45.952683519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:56:46.853442972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:56:46.858139697Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:56:46.858169172Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:56:46.85819551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:56:46.86183066Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:56:48.925204156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:56:48.932777747Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:56:49.11700449Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:56:49.117842286Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:56:49.134185769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:56:49.134221262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:56:49.134233099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:56:49.134228508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:49.134301075Z caller=checkpoint.go:372 msg="writing series" size="132 kB"
ts=2026-05-10T19:56:49.253217472Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T19:56:49.450590114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:56:49.458759769Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:56:51.443654655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:56:51.44742111Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T19:56:52.467655423Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=debug ts=2026-05-10T19:56:52.467678891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-05-10T19:56:54.134361044Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:56:55.40627281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:56:55.410246979Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:56:55.619911195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:56:55.623657683Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:56:55.801158329Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T19:56:57.070330246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:56:57.074147247Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=warn ts=2026-05-10T19:56:59.134322254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:56:59.13442982Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T19:56:59.97409019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:56:59.977657702Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:57:01.273698078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:57:01.277191303Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:57:01.277216641Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:57:01.277238979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:57:01.280137915Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T19:57:02.470170875Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T19:57:02.470202244Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=3
level=info ts=2026-05-10T19:57:03.883867913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:57:03.888514362Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T19:57:04.134101107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:05.801529409Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-05-10T19:57:09.134059463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:09.134127255Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-10T19:57:10.049768646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:57:10.05419143Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:57:10.054224373Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:57:10.054249026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:57:10.057584233Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:57:11.981170052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:57:11.988103673Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:57:12.467817592Z caller=checkpoint.go:372 msg="writing series" size="571 kB"
level=info ts=2026-05-10T19:57:12.687226526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:57:12.690969394Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T19:57:14.134475923Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:15.800760102Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T19:57:15.800781226Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T19:57:15.894859368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:57:15.901270746Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T19:57:16.965763892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:57:16.979343868Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
ts=2026-05-10T19:57:17.943178713Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52966"
level=info ts=2026-05-10T19:57:19.133387687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:57:19.133420199Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:57:19.13342944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:57:19.133388871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:57:19.13724568Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-05-10T19:57:19.138932927Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005201749s
ts=2026-05-10T19:57:19.255942065Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:57:20.24533795Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:57:20.279350899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:57:20.282911841Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:57:20.458561492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:57:20.462058661Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:57:20.462079586Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:57:20.462101899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:57:20.464955859Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:57:22.476264992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:57:22.485441306Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:57:24.134233627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:27.681603525Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T19:57:29.133431303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:57:29.596095614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:57:29.600169786Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:57:30.216873498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:57:30.221011294Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T19:57:30.310219435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:57:30.317342363Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-05-10T19:57:33.724840401Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46244"
level=warn ts=2026-05-10T19:57:34.134389795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:57:36.031234521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:57:36.035146305Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:57:36.713738127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:57:36.71740206Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-10T19:57:36.717422294Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:57:36.717444105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:57:36.720224906Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T19:57:39.133678167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:57:39.997777388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:57:40.001360983Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:57:41.441674145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:57:41.445567837Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:57:42.728547216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:57:42.731943227Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T19:57:43.046686727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:57:43.050075184Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:57:43.105136786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:57:43.108298865Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T19:57:44.134442046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:57:45.167545267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:56886"
level=debug ts=2026-05-10T19:57:45.780621315Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:57:46.263823738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:57:46.267230767Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T19:57:47.658235799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:57:47.66320152Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:57:47.663224573Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:57:47.663246934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:57:47.666162195Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-10T19:57:47.945556451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:60624"
level=info ts=2026-05-10T19:57:48.207702009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:57:48.216659752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:57:48.864259566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:57:48.86753435Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:57:49.117262712Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:57:49.117690629Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:57:49.134030943Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:57:49.134061948Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:57:49.134070931Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T19:57:49.134080391Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T19:57:49.134083189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:57:49.134202773Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-10T19:57:49.13421063Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T19:57:49.134226913Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.395µs success=true
level=debug ts=2026-05-10T19:57:49.134562836Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T19:57:49.258879564Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:57:52.468603646Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T19:57:54.134434225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:55.803250583Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T19:57:55.803274224Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-05-10T19:57:59.13380126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:57:59.134814239Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T19:58:02.221393285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:58:02.231967511Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T19:58:02.468765149Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
ts=2026-05-10T19:58:03.72738577Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:34148"
level=info ts=2026-05-10T19:58:03.858973174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:58:03.863091569Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:58:04.099718482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:58:04.108908667Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:58:04.134379444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:05.803867411Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-05-10T19:58:05.803891493Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=3
level=info ts=2026-05-10T19:58:07.310217834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:58:07.318147619Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T19:58:09.134141227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:09.13525312Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=info ts=2026-05-10T19:58:09.991106141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:58:09.994620289Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T19:58:11.321579378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:58:11.325024531Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T19:58:11.348241857Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:58:11.758448217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:58:11.76541918Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:58:11.867203882Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T19:58:12.468214811Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-10T19:58:14.133769765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:58:15.671849987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:58:15.675623726Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T19:58:15.801155909Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T19:58:19.133568652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:58:19.133585647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:58:19.13360765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:58:19.133615779Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:58:19.13480492Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
ts=2026-05-10T19:58:19.262327721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T19:58:22.468530254Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-05-10T19:58:22.704224631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:58:22.711071354Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T19:58:24.134120021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:25.80237089Z caller=checkpoint.go:372 msg="writing series" size="581 kB"
level=info ts=2026-05-10T19:58:28.081212792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:58:28.084947886Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T19:58:29.13356941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:29.134883768Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=debug ts=2026-05-10T19:58:29.13490505Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-05-10T19:58:29.373849825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:58:29.381709121Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:58:29.381736892Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:58:29.381785355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:58:29.389166677Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:58:30.459082286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:58:30.462929644Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:58:32.468190299Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
ts=2026-05-10T19:58:33.730797422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40356"
level=warn ts=2026-05-10T19:58:34.133644517Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:58:34.285965312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:58:34.289913844Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T19:58:34.624375622Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:58:34.628281011Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=debug ts=2026-05-10T19:58:35.801198005Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T19:58:36.26272228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:58:36.266311199Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T19:58:36.362770078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:58:36.366458629Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:58:38.827231402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:58:38.835240852Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T19:58:39.133871982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:39.134937059Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T19:58:39.523766551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:58:39.527676911Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T19:58:39.552868875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="503 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:58:39.561119131Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T19:58:42.468174898Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=warn ts=2026-05-10T19:58:44.134322621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:45.117190801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T19:58:45.172264223Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:60328"
level=debug ts=2026-05-10T19:58:45.801793083Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=info ts=2026-05-10T19:58:49.117244773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:58:49.117702767Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:58:49.134083275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:58:49.134107369Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:58:49.134116069Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:58:49.134171257Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:49.13526464Z caller=checkpoint.go:372 msg="writing series" size="7.9 kB"
ts=2026-05-10T19:58:49.264744988Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T19:58:51.455715062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:58:51.465822855Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:58:52.468382364Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=warn ts=2026-05-10T19:58:54.13379305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:58:54.609647763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:58:54.618103199Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T19:58:55.76642881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:58:55.775951094Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T19:58:55.802406304Z caller=checkpoint.go:372 msg="writing series" size="652 kB"
level=debug ts=2026-05-10T19:58:55.802430514Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-10T19:58:56.788608948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:58:56.793158895Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T19:58:59.133790786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:58:59.134963928Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-10T19:59:02.106445295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:59:02.113075014Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T19:59:02.4682244Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-05-10T19:59:04.133684003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:05.21419912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T19:59:05.21612151Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T19:59:05.801857355Z caller=checkpoint.go:372 msg="writing series" size="369 kB"
level=info ts=2026-05-10T19:59:07.109929047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="527 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:59:07.117936072Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:59:07.122017086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T19:59:07.125455964Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T19:59:09.133402738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:09.135763323Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-10T19:59:09.377332852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:59:09.381100376Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T19:59:12.484583807Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:59:12.484610196Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=5
level=warn ts=2026-05-10T19:59:14.134035239Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:15.805904187Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-05-10T19:59:15.805928207Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=1
level=warn ts=2026-05-10T19:59:19.133789942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T19:59:19.133786709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:59:19.13381849Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:59:19.133826536Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T19:59:19.150252543Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T19:59:19.150277617Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-05-10T19:59:19.26761759Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:59:19.830879637Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T19:59:20.243125848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T19:59:20.249795976Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T19:59:20.707649237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:59:20.712136177Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:59:20.712160805Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T19:59:20.712188342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:59:20.71606508Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:59:20.804173058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T19:59:20.808286667Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T19:59:22.468923062Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=info ts=2026-05-10T19:59:23.729054463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T19:59:23.733304785Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T19:59:24.133567172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:25.801427212Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T19:59:25.801453013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T19:59:26.283449219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T19:59:26.28873925Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T19:59:27.321074478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T19:59:27.324536325Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T19:59:29.133736551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:29.134944184Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=info ts=2026-05-10T19:59:30.040608399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T19:59:30.044129713Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T19:59:31.586881241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T19:59:31.590630928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T19:59:32.468583903Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-10T19:59:34.133672658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:35.80190658Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-10T19:59:36.033661479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T19:59:36.042717349Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T19:59:36.116965295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T19:59:36.121794921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T19:59:38.901664729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T19:59:38.91213103Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T19:59:39.133405021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:39.214059144Z caller=checkpoint.go:372 msg="writing series" size="66 MB"
level=debug ts=2026-05-10T19:59:39.214091069Z caller=checkpoint.go:384 msg="flushing series" totalSize="66 MB" series=4
level=info ts=2026-05-10T19:59:40.27221559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T19:59:40.294432362Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T19:59:42.469665483Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-10T19:59:43.120462269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T19:59:43.136883805Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T19:59:44.134167714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T19:59:45.179155448Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:40290"
level=debug ts=2026-05-10T19:59:45.802299907Z caller=checkpoint.go:372 msg="writing series" size="686 kB"
ts=2026-05-10T19:59:47.953939491Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38900"
level=info ts=2026-05-10T19:59:49.118844276Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T19:59:49.120545438Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T19:59:49.133661299Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T19:59:49.133704771Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T19:59:49.133717019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T19:59:49.133824769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:49.140887101Z caller=checkpoint.go:372 msg="writing series" size="181 kB"
ts=2026-05-10T19:59:49.270831847Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T19:59:50.089725571Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T19:59:52.468324232Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-05-10T19:59:52.560024318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T19:59:52.569340774Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T19:59:54.134383766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:55.802097201Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=info ts=2026-05-10T19:59:55.852329608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T19:59:55.856742465Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T19:59:56.659473711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="527 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T19:59:56.66855917Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T19:59:57.674076137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T19:59:57.677838148Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T19:59:59.134199571Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T19:59:59.135483917Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T19:59:59.135500602Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-05-10T20:00:00.173327528Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:00:01.480427965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:00:01.484622152Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:00:02.330925177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:00:02.336950382Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:00:02.468163829Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=warn ts=2026-05-10T20:00:04.134149254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:00:04.241617924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:00:04.245859122Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:00:05.80153041Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-05-10T20:00:06.657457429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:00:06.662848103Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:00:07.476464621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:00:07.48047297Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:00:07.480496837Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:00:07.480522075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:00:07.483886384Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:00:09.135237064Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=warn ts=2026-05-10T20:00:09.135310536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:12.468824366Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=warn ts=2026-05-10T20:00:14.140011128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:15.801440159Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T20:00:15.906778386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:00:15.911910542Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:00:16.827852179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:00:16.833833567Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:00:17.861489428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:00:17.87006702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T20:00:17.957226783Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:48748"
level=info ts=2026-05-10T20:00:19.133863267Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:00:19.133896915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:00:19.133906Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:00:19.13398633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:19.135079465Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
ts=2026-05-10T20:00:19.276732205Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:00:20.540848465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T20:00:20.545127812Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=info ts=2026-05-10T20:00:20.868862962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:00:20.872378724Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:00:22.493207506Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-05-10T20:00:22.493241854Z caller=checkpoint.go:384 msg="flushing series" totalSize="21 MB" series=7
level=debug ts=2026-05-10T20:00:24.10817879Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:00:24.133939231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:00:24.873478112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:00:24.880437753Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:00:25.802019564Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-10T20:00:26.744988229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:00:26.750354916Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:00:29.133968921Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:29.13504649Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T20:00:30.955030825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:00:30.965919765Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:00:32.469334387Z caller=checkpoint.go:372 msg="writing series" size="753 kB"
ts=2026-05-10T20:00:33.74014204Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:47686"
level=warn ts=2026-05-10T20:00:34.1343759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:35.801307645Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-05-10T20:00:37.713454183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="527 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:00:37.723070525Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:00:39.036050651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:00:39.044729106Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:00:39.133470622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:39.134674893Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:00:41.368558763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:00:41.372397183Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:00:42.468108605Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:00:42.616616833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:00:42.621453266Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T20:00:44.133737365Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:00:45.690708787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:00:45.699477675Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:00:45.802319083Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T20:00:45.802342485Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=7
level=info ts=2026-05-10T20:00:46.220118448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:00:46.223744443Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-05-10T20:00:47.960314059Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34822"
level=info ts=2026-05-10T20:00:49.11702022Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:00:49.117164748Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T20:00:49.118438739Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T20:00:49.118479224Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976047 n=1
level=debug ts=2026-05-10T20:00:49.118520268Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976047 n=1
level=info ts=2026-05-10T20:00:49.133488354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:00:49.133508248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:49.133525777Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:00:49.133538041Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T20:00:49.134621999Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-05-10T20:00:49.279496692Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:00:50.07907395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:00:50.082813523Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:00:52.468015825Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=warn ts=2026-05-10T20:00:54.133853371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:00:54.312752404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:00:54.316835969Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:00:55.549913645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:00:55.553309514Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:00:55.80224413Z caller=checkpoint.go:372 msg="writing series" size="5.0 kB"
level=debug ts=2026-05-10T20:00:56.793103892Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:00:59.134381775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:00:59.134423236Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-05-10T20:01:00.097926502Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:01:00.587385464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:01:00.591998014Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:01:02.469248416Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-10T20:01:02.469272666Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-10T20:01:03.921598492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=676ab56d031ecf26 immediate=false num_chunks=1 total_comp="648 B" avg_comp="648 B" total_uncomp="1.5 kB" avg_uncomp="1.5 kB" full=1 labels="{stream=\"project-controller\"}"
level=error ts=2026-05-10T20:01:03.925376471Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/676ab56d031ecf26/19e133397b3%3A19e133397b4%3A7f287284?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"project-controller\"}"
level=error ts=2026-05-10T20:01:03.925408033Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:01:03.925456846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:01:03.930863392Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T20:01:04.133798225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:05.801978671Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:01:08.281923097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:01:08.288469567Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:01:09.13427493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:09.13829706Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:01:09.138329726Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-10T20:01:10.457054634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:01:10.460995763Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:01:11.174183941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:01:11.183529789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:01:12.468258247Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-10T20:01:14.133571202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:01:14.5053007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:01:14.513073128Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:01:14.513101482Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:01:14.513135165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:01:14.519478026Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:01:15.802176008Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-10T20:01:17.963380087Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:46942"
level=warn ts=2026-05-10T20:01:19.134434574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:01:19.134682789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:01:19.134725141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:01:19.13473965Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-05-10T20:01:19.134786795Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
ts=2026-05-10T20:01:19.282336531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:01:20.154237233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:01:20.163189359Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:01:20.167157097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:01:20.171852231Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T20:01:21.184716764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:01:21.189112737Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:01:22.468875562Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-05-10T20:01:24.13415532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:01:24.478942659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:01:24.488445808Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:01:25.801682902Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-10T20:01:28.576258634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:01:28.586201127Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:01:29.134339861Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:29.135465578Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-05-10T20:01:30.82388738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:01:30.828213483Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:01:31.59252478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:01:31.600055597Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:01:32.468997768Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-05-10T20:01:34.134046959Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:34.377458712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:01:34.37753654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="37 kB" avg_uncomp="37 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:01:34.381377547Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:01:35.013495352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:01:35.017281504Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:01:35.343084463Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:01:35.801803474Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T20:01:35.80183243Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=info ts=2026-05-10T20:01:36.770831446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:01:36.774990297Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:01:39.133799601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:39.135951276Z caller=checkpoint.go:372 msg="writing series" size="811 kB"
level=info ts=2026-05-10T20:01:39.820510246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:01:39.824847079Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T20:01:40.745708276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:01:40.751115178Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:01:42.411005617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:01:42.419146726Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:01:42.419174865Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:01:42.41921157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:01:42.427425287Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:01:42.469014663Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T20:01:42.469045513Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=warn ts=2026-05-10T20:01:44.134043317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:45.802203172Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T20:01:45.876357006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:01:45.881859726Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-10T20:01:47.965965272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56070"
level=info ts=2026-05-10T20:01:49.11720473Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:01:49.117665688Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:01:49.13405281Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:01:49.134093146Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:01:49.134102561Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:01:49.134097337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:49.135267532Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
ts=2026-05-10T20:01:49.284956578Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:01:52.468114938Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-10T20:01:52.613555962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:01:52.618054524Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:01:54.055120094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:01:54.059316089Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:01:54.133860581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:55.802141069Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-10T20:01:55.815376986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:01:55.822986978Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-05-10T20:01:59.134291266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:01:59.135377968Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:02:00.387696565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:02:00.394777994Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:02:01.795375321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:02:01.803475854Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:02:02.470516072Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-10T20:02:02.470542551Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=6
level=warn ts=2026-05-10T20:02:04.133597737Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:05.164693008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:02:05.170586279Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:02:05.545785394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:02:05.554863583Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:02:05.801545966Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-10T20:02:07.124122287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:02:07.132660571Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:02:07.746219795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:02:07.749907047Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:02:07.842244834Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:02:08.677292638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:02:08.686157151Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:02:08.686186765Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:02:08.686220145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:02:08.694569332Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:02:09.134066991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:02:09.135298504Z caller=checkpoint.go:372 msg="writing series" size="160 kB"
level=info ts=2026-05-10T20:02:09.232494778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:02:09.241395767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:02:09.24142401Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:02:09.241450568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:02:09.250377935Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:02:10.642870122Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:02:12.468733473Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
level=warn ts=2026-05-10T20:02:14.133667115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:15.093765929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:02:15.097990606Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:02:15.802249083Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T20:02:15.802272191Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-10T20:02:17.590782569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:02:17.595729798Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:02:19.133483582Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:19.133511453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:02:19.133540055Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:02:19.13355095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:02:19.288292696Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:02:19.376148325Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-05-10T20:02:19.378975163Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.244622347s
level=info ts=2026-05-10T20:02:20.939554191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:02:20.94480293Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:02:23.040407319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:02:23.046370043Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T20:02:23.990699871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:02:23.994258941Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:02:23.994282134Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:02:23.994309051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:02:23.997555645Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:02:24.13376078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:25.01447157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:02:25.019446321Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:02:27.016714307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:02:27.024080586Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:02:27.633582272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:02:27.642696509Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:02:27.966528545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=7 total_comp="18 MB" avg_comp="2.6 MB" total_uncomp="116 MB" avg_uncomp="17 MB" full=6 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:02:27.974520915Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T20:02:29.133801487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:29.648961796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:02:29.652651656Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:02:32.559925174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:02:32.564443516Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T20:02:34.134173031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:02:38.607154541Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:02:39.134141193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:42.74676775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:02:42.757203905Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:02:44.133671543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:02:45.191438174Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57220"
level=info ts=2026-05-10T20:02:45.460795385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:02:45.467020229Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T20:02:49.116996738Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:02:49.117415903Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:02:49.133739014Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:02:49.133781277Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:02:49.133792836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T20:02:49.1338033Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T20:02:49.133861216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:49.133958553Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-10T20:02:49.133968389Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T20:02:49.133991439Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.982µs success=true
level=debug ts=2026-05-10T20:02:49.134347349Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T20:02:49.291524519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:02:50.732770313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:02:50.741308284Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:02:50.741335849Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:02:50.741365433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:02:50.749229842Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:02:52.387550991Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T20:02:53.588135669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:02:53.59283077Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:02:53.636016601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:02:53.643509687Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T20:02:54.134436547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:02:55.641647735Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-05-10T20:02:55.641673677Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-05-10T20:02:57.973465875Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:02:57.975361913Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T20:02:58.893804835Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:02:59.134103214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:02:59.726577246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:02:59.732767136Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:03:01.121645744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:03:01.125352549Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:03:01.741024226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:03:01.745146492Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:03:02.146644504Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-05-10T20:03:03.708853595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:03:03.712882323Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-10T20:03:03.749548985Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:44408"
level=warn ts=2026-05-10T20:03:04.133992614Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:03:04.782375886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:03:04.789898203Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:03:05.269219473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:03:05.277210446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:03:05.402698155Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-05-10T20:03:05.402717159Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=3
level=info ts=2026-05-10T20:03:06.733484328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:03:06.736913301Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:03:07.032224069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:03:07.042494919Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:03:07.671114038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:03:07.674653391Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:03:08.291601812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:03:08.298864228Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:03:08.653025472Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-10T20:03:09.134182977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:03:10.914521713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:03:10.91881227Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:03:11.906359128Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-05-10T20:03:11.914944833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:03:13.764548816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:03:13.768221746Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:03:14.133702391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:15.158622886Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=info ts=2026-05-10T20:03:15.497275099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:03:15.500798915Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T20:03:18.411793636Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:03:19.134283418Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:03:19.13431065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:03:19.134319593Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:03:19.13433945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:03:19.294838638Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:03:21.66505948Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T20:03:23.514743566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:03:23.518754975Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:03:23.88853838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:03:23.89383977Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T20:03:24.134264621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:24.918159388Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=debug ts=2026-05-10T20:03:28.171092362Z caller=checkpoint.go:372 msg="writing series" size="478 kB"
level=info ts=2026-05-10T20:03:28.835691293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:03:28.843581934Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T20:03:29.134000578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:31.424030651Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-10T20:03:31.424052394Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-05-10T20:03:34.133840663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:34.677130744Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=info ts=2026-05-10T20:03:34.680226183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:03:34.686065308Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:03:34.686094375Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:03:34.686120068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:03:34.690015685Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:03:37.930452485Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-10T20:03:39.133760967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:03:40.280266361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:03:40.287960907Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:03:41.182740086Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=info ts=2026-05-10T20:03:41.804968225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:03:41.810080058Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T20:03:42.535196926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:03:42.540886859Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:03:43.607817596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:03:43.611414025Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:03:43.611434063Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:03:43.611456265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:03:43.614357491Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T20:03:44.134295149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:03:44.422776873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="378 B" avg_comp="378 B" total_uncomp="14 kB" avg_uncomp="14 kB" idle=1 labels="{stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:03:44.422828078Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-05-10T20:03:44.426459795Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:03:44.43548498Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-10T20:03:46.772516645Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:03:47.689366227Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-05-10T20:03:49.116964024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:03:49.116972748Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-05-10T20:03:49.133757673Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:03:49.13378135Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:49.133793737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:03:49.133805232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:03:49.297664159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:03:50.942109756Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-10T20:03:51.27056372Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:03:51.274099584Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:03:52.063596529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:03:52.066910564Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:03:54.133910589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:03:54.195487042Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=info ts=2026-05-10T20:03:54.990042795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:03:54.997737453Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:03:55.656744844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:03:55.660583756Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:03:57.448814609Z caller=checkpoint.go:372 msg="writing series" size="568 kB"
level=debug ts=2026-05-10T20:03:57.448836705Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-10T20:03:59.134096691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:03:59.414005003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:03:59.418218289Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:04:00.701341099Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=info ts=2026-05-10T20:04:00.800768283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:04:00.804228665Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:04:01.190465965Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:04:01.193901751Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:04:01.193922533Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:01.193945284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:04:01.196715944Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:04:02.117221955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:04:02.124620127Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:04:03.565996511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:04:03.569415782Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:04:03.569440369Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:03.569463292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:04:03.57237883Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-10T20:04:03.755582334Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:59458"
level=debug ts=2026-05-10T20:04:03.954402807Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-05-10T20:04:04.134347385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:04.748342832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:04:04.75625852Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:04:05.031037479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:04:05.034323606Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:04:05.034345438Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:05.034365717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:04:05.03709104Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:04:05.434405351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:04:05.437755563Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:04:05.43777658Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:05.437798175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:04:05.440412886Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:04:05.918905639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:04:05.922951707Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:04:06.146081427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:04:06.149737077Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-10T20:04:06.149759364Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:06.149789155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:04:06.152664547Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T20:04:07.207063244Z caller=checkpoint.go:372 msg="writing series" size="489 kB"
level=warn ts=2026-05-10T20:04:09.133843715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:04:10.460374115Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=debug ts=2026-05-10T20:04:10.460397198Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-10T20:04:11.08481361Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:04:11.084874729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="574 kB" avg_comp="574 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:04:11.089060779Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:04:13.72931233Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-10T20:04:13.729334714Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=warn ts=2026-05-10T20:04:14.133665973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:14.402527962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:04:14.405939178Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:04:15.219684155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:04:15.223022886Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:04:16.026152501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:04:16.02956902Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:04:16.970417963Z caller=checkpoint.go:372 msg="writing series" size="4.5 MB"
level=debug ts=2026-05-10T20:04:16.970440503Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=1
level=info ts=2026-05-10T20:04:17.273156839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:04:17.2773802Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:04:19.133751395Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:04:19.133782837Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:04:19.133790893Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:04:19.133790045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:04:19.300412422Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:04:19.740063585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:04:19.74375709Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:04:19.95531717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:04:19.958722273Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T20:04:20.231700509Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T20:04:20.231733389Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-10T20:04:22.465239906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:04:22.470394274Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T20:04:22.842542365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:04:22.850667548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:04:23.408357818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:04:23.413014883Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:04:23.472693071Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T20:04:24.133584028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:24.33136505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:04:24.338434084Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:04:24.33845403Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:04:24.338494155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:04:24.345507151Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:04:26.725431957Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T20:04:26.725455278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T20:04:26.880787364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:04:26.885816948Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:04:29.134087818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:29.760678535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:04:29.764255702Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:04:29.977948454Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-05-10T20:04:30.791119868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:04:30.797059798Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:04:32.109175069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:04:32.112739998Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:04:33.231121661Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-05-10T20:04:34.133752636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:04:35.942163397Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:04:36.391462801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:04:36.400072958Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:04:36.484692819Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-05-10T20:04:39.133561196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:04:39.818237532Z caller=checkpoint.go:372 msg="writing series" size="70 MB"
level=debug ts=2026-05-10T20:04:39.818273765Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 MB" series=4
level=debug ts=2026-05-10T20:04:42.990453185Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-10T20:04:44.133992315Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:44.6870351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:04:44.697393867Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T20:04:46.18917989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:04:46.200040307Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:04:46.243734172Z caller=checkpoint.go:372 msg="writing series" size="507 kB"
level=info ts=2026-05-10T20:04:47.620318939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:04:47.637386786Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-10T20:04:47.97741497Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38080"
level=debug ts=2026-05-10T20:04:48.637732724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:04:49.118619224Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:04:49.119282253Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:04:49.133342936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:04:49.133366991Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:04:49.133374774Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:04:49.13340829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:04:49.303071318Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:04:49.496262391Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=info ts=2026-05-10T20:04:49.65924431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:04:49.668021883Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:04:50.443498449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:04:50.462321957Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:04:51.307951837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:04:51.321680788Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:04:51.413479607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:04:51.422733405Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:04:52.36090384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:04:52.377830101Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:04:52.749199058Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-05-10T20:04:54.133708755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:04:55.647245117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:04:55.651809575Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:04:56.001793578Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-10T20:04:57.244822871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:04:57.25005186Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:04:57.518069785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:04:57.52145208Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:04:57.642692715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:04:57.646871939Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-10T20:04:59.133833707Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:04:59.255420166Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T20:04:59.255441129Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T20:05:00.154363833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:05:00.157748884Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:05:01.795590735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:05:01.803556144Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:05:02.508333668Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-10T20:05:04.134088922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:05:05.761095175Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-10T20:05:06.748672524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:05:06.75419303Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-05-10T20:05:07.023452647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:05:07.031385893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:05:09.014108732Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T20:05:09.133750379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:05:11.451981198Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:05:12.267470803Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T20:05:12.75556163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:05:12.759222786Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:05:14.133564373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:05:15.520320223Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=debug ts=2026-05-10T20:05:18.773098989Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T20:05:19.134049878Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:05:19.134078119Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:05:19.134087284Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:05:19.134167334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:05:19.306504833Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:05:21.108917498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:05:21.115466447Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:05:22.026504679Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T20:05:24.133593401Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:05:25.30714374Z caller=checkpoint.go:372 msg="writing series" size="23 MB"
level=debug ts=2026-05-10T20:05:25.307168744Z caller=checkpoint.go:384 msg="flushing series" totalSize="24 MB" series=8
level=info ts=2026-05-10T20:05:26.536291578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:05:26.540078616Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:05:28.532880213Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T20:05:29.134254572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:05:29.54567532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:05:29.550818512Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:05:29.690201749Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:05:29.693667356Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:05:31.241691792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:05:31.246273802Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:05:31.785726138Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:05:34.134074045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:05:35.038531331Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-05-10T20:05:37.663761965Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:05:38.291769311Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-05-10T20:05:38.497219685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:05:38.500906281Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T20:05:39.133622688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:05:40.055238939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:05:40.058968051Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T20:05:41.544435067Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:05:42.397376556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:05:42.402785919Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:05:42.402809647Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:05:42.402833743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:05:42.40728623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:05:42.536676502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:05:42.54448706Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:05:43.821304651Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:05:44.133782319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:05:44.275023044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:05:44.278435034Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:05:44.797774355Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:05:45.276154701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:05:45.280418834Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:05:46.347552394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:05:46.356680477Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:05:47.223868061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:05:47.227417407Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:05:48.050540237Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T20:05:48.050564203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T20:05:49.116971208Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:05:49.117398064Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:05:49.132690161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:05:49.133993373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:05:49.134042079Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:05:49.134053277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:05:49.134104298Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-05-10T20:05:49.141090698Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T20:05:49.30930743Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:05:49.988241876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:05:49.993392385Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T20:05:51.303323368Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:05:52.379380135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:05:52.383379588Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:05:54.134133222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:05:54.40959215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:05:54.413515767Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:05:54.55666441Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=info ts=2026-05-10T20:05:54.661819287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:05:54.671412677Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:05:56.465094884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:05:56.472974272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:05:56.729196735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:05:56.732883988Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:05:57.559911239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:05:57.564335384Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:05:57.809216873Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=warn ts=2026-05-10T20:05:59.133897013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:01.062799906Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-05-10T20:06:04.134059968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:04.317381793Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:06:04.317407488Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-05-10T20:06:07.568513606Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-10T20:06:09.13347363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:10.826305109Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:06:10.826328434Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-10T20:06:11.108035582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:06:11.115570314Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:06:12.291769174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:06:12.295808657Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:06:13.121569647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:06:13.129528133Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:06:14.074234641Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-10T20:06:14.134349101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:17.328848927Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:06:19.133506043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:06:19.13354078Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:06:19.133551468Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:06:19.133675014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:06:19.312335008Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:06:19.453347373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:06:20.580625959Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-10T20:06:21.571761982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:06:21.576089076Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:06:23.834154281Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-05-10T20:06:24.133843901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:06:26.531706205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:06:26.535709868Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:06:27.086670875Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-10T20:06:27.91022401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:06:27.914015367Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:06:27.955094473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:06:27.958899039Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T20:06:28.613898103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:06:28.618359872Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:06:29.133431447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:30.339341105Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=debug ts=2026-05-10T20:06:30.469328017Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:06:33.593147483Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-05-10T20:06:33.766427728Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:40512"
level=warn ts=2026-05-10T20:06:34.133986544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:06:34.259297793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:06:34.263139384Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:06:34.263163642Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:06:34.263188929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:06:34.266033845Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:06:34.78815904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:06:34.792702339Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:06:36.845562984Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T20:06:36.845586577Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=info ts=2026-05-10T20:06:38.230853111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:06:38.239665983Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:06:38.721834699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:06:38.732525035Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:06:39.133415578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:06:39.922894042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:06:39.927507538Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:06:39.950570478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:06:39.953784895Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:06:40.099845722Z caller=checkpoint.go:372 msg="writing series" size="976 kB"
level=info ts=2026-05-10T20:06:41.192754379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:06:41.201809626Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:06:43.351814528Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T20:06:43.351838684Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-10T20:06:44.134164275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:06:44.702405096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:06:44.707154046Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:06:45.335275672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:06:45.340428434Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-05-10T20:06:46.604882187Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-10T20:06:47.74589274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:06:47.754469336Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:06:49.117059238Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:06:49.117087008Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:06:49.133362007Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:06:49.133381984Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:06:49.133391055Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:06:49.133416467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:06:49.315349155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:06:49.465770718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:06:49.474357041Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:06:49.857841543Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=debug ts=2026-05-10T20:06:50.681144881Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:06:50.971001104Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:06:50.975051724Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:06:53.110856255Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-10T20:06:53.677264347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:06:53.680901407Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T20:06:54.133764656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:06:56.363567526Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-10T20:06:59.134132513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:06:59.522924203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:06:59.52640296Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:06:59.617030184Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:07:02.326184517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:07:02.329849247Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:07:02.872199897Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T20:07:02.872223581Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=warn ts=2026-05-10T20:07:04.134530306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:07:06.122992959Z caller=checkpoint.go:372 msg="writing series" size="325 kB"
level=info ts=2026-05-10T20:07:07.590758567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:07:07.597739617Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:07:09.034801084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:07:09.038232021Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T20:07:09.133496991Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:07:09.375797284Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
level=info ts=2026-05-10T20:07:09.464534545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:07:09.468439667Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:07:10.179693386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:07:10.183626614Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:07:12.628774703Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
level=debug ts=2026-05-10T20:07:12.62879999Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T20:07:13.177927412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:07:13.183475581Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-05-10T20:07:14.134399215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:07:15.881557548Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
ts=2026-05-10T20:07:17.989347933Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54860"
level=debug ts=2026-05-10T20:07:18.11972696Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:07:18.121391721Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T20:07:18.939507078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:07:18.943184944Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:07:19.134317004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:07:19.134333194Z caller=checkpoint.go:384 msg="flushing series" totalSize="259 kB" series=1
level=debug ts=2026-05-10T20:07:19.134348375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:07:19.134360571Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:07:19.134427848Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:19.141663257Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-05-10T20:07:19.143968517Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009808539s
ts=2026-05-10T20:07:19.318476379Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:07:19.392287587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:07:19.399899747Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:07:23.351026927Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:07:23.958470236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:07:23.968073252Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T20:07:24.133523514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:25.648749956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:07:25.653087828Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:07:25.66214009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:07:25.665338575Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:07:26.925213692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:07:26.928910737Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:07:29.133811878Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:30.08261032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:07:30.086389487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:07:31.491480688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:07:31.495717063Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:07:33.583660752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="884 kB" avg_comp="884 kB" total_uncomp="7.1 MB" avg_uncomp="7.1 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-05-10T20:07:33.58842014Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19e133397b3%3A19e1360b8ae%3Add963a63?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-05-10T20:07:33.588441324Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:07:33.588463631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:07:33.592849134Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-10T20:07:34.134160908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:37.164752626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:07:37.172254769Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:07:39.134269945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:42.278927268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:07:42.283010295Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:07:42.536332603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:07:42.539890658Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:07:44.134362641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:44.782322601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:07:44.790372724Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T20:07:45.211382965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:33768"
level=info ts=2026-05-10T20:07:46.467910082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:07:46.474010641Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:07:47.903105498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:07:47.911101373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:07:49.117108758Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:07:49.117117474Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:07:49.13346021Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T20:07:49.133472988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:07:49.133495116Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:07:49.133503819Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:07:49.13351617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:49.133604076Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-10T20:07:49.133612781Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T20:07:49.133646936Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.59µs success=true
level=debug ts=2026-05-10T20:07:49.134140289Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T20:07:49.321336655Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:07:52.387406287Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T20:07:54.134198364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:07:54.324290938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:07:54.327905664Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:07:54.756515613Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:07:55.640844804Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:07:55.640868333Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-10T20:07:58.157175985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:07:58.16054237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-05-10T20:07:58.893503303Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:07:59.134002574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:00.482756243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:08:00.486418893Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:08:00.486443838Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:08:00.486465831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:08:00.489854772Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:08:02.146911796Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=warn ts=2026-05-10T20:08:04.133904687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:04.277182889Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:08:04.280614569Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:08:04.508288233Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:08:05.119867147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:08:05.125565622Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:08:05.125588037Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:08:05.125611321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:08:05.131148945Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:08:05.40047329Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-05-10T20:08:05.400495375Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=info ts=2026-05-10T20:08:05.571607085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:08:05.579112161Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:08:07.080684609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:08:07.084242325Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:08:08.652076008Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-10T20:08:09.133805303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:10.271853824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:08:10.275401955Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:08:11.905149877Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-10T20:08:13.981533169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:08:13.989807813Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:08:13.989827843Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:08:13.989857177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:08:13.997576161Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T20:08:14.134067078Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:15.158722181Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-10T20:08:16.188482596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:08:16.194966625Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:08:17.927561331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:08:17.932343314Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:08:18.411014765Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:08:18.550403502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:08:18.553871863Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:08:19.133686396Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:08:19.133693111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:19.133718903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:08:19.133728328Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:08:19.323863227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:08:20.045753787Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:08:20.049499137Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:08:21.245995696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:08:21.250667625Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:08:21.319906183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:08:21.323323258Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:08:21.664607049Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-05-10T20:08:24.134247205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:24.137399039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:08:24.145789464Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:08:24.57091733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:08:24.574373128Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:08:24.917569209Z caller=checkpoint.go:372 msg="writing series" size="8.8 kB"
level=info ts=2026-05-10T20:08:25.018836723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:08:25.025800037Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:08:28.171305878Z caller=checkpoint.go:372 msg="writing series" size="598 kB"
level=debug ts=2026-05-10T20:08:28.171328041Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-10T20:08:28.331220145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:08:28.335074268Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:08:28.72126707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:08:28.728759941Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T20:08:29.134289967Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:30.700852912Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:08:31.423388782Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-05-10T20:08:32.432210872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:08:32.440111959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:08:34.133570512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:34.676282183Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-05-10T20:08:37.929103318Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-10T20:08:39.133835542Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:39.851028366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:08:39.855267803Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:08:40.187208095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:08:40.191056235Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:08:41.182083621Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=warn ts=2026-05-10T20:08:44.13358115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:44.435084491Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-10T20:08:44.541103618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:08:44.550132941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T20:08:45.215903226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:38622"
level=debug ts=2026-05-10T20:08:47.688446618Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=info ts=2026-05-10T20:08:49.116984213Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:08:49.117332732Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:08:49.133613352Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:08:49.133646245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:08:49.133654086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:08:49.133716533Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:08:49.326710985Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:08:50.258836318Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:08:50.941938736Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-10T20:08:53.446174933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:08:53.45007921Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-05-10T20:08:53.488096876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:08:53.493555679Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-05-10T20:08:54.134248088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:08:54.194864763Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=info ts=2026-05-10T20:08:55.230625233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:08:55.234075948Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:08:57.447988486Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=debug ts=2026-05-10T20:08:57.448011997Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=9
level=info ts=2026-05-10T20:08:58.647041148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:08:58.650803135Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-10T20:08:59.133818508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:08:59.964501442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:08:59.971471309Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:09:00.700616834Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=debug ts=2026-05-10T20:09:02.358678166Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:09:02.590198655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:09:02.593645026Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:09:03.953788767Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=warn ts=2026-05-10T20:09:04.134215275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:09:05.932549569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:09:05.939982588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:09:06.364165131Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:09:06.367780095Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:09:06.451015938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:09:06.459392163Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:09:06.4594147Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:09:06.459441281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:09:06.466234457Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:09:07.151841991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:09:07.156493733Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:09:07.206934998Z caller=checkpoint.go:372 msg="writing series" size="586 kB"
level=debug ts=2026-05-10T20:09:07.206962845Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T20:09:07.328127623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:09:07.331720658Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:09:07.331739901Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:09:07.331760652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:09:07.334673601Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:09:07.787855016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:09:07.792335618Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-10T20:09:09.134298625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:09:09.134299592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:09:09.138901671Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:09:10.459916399Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=info ts=2026-05-10T20:09:10.59615118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:09:10.599662965Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:09:11.963237312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:09:11.967662527Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:09:13.72730937Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-05-10T20:09:13.727333575Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=warn ts=2026-05-10T20:09:14.133802133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:09:16.969794274Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-05-10T20:09:16.969818419Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=1
level=info ts=2026-05-10T20:09:17.585840653Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:09:17.589612032Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-10T20:09:17.997658241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54786"
level=info ts=2026-05-10T20:09:19.134092592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:09:19.134131483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:09:19.134140393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:09:19.134153102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:09:19.329107896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:09:20.226887301Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T20:09:20.226911979Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-05-10T20:09:23.472612854Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=info ts=2026-05-10T20:09:23.544555952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:09:23.5525055Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:09:24.133592657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:09:25.400761938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:09:25.409153206Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:09:26.724985572Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T20:09:26.725009175Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-10T20:09:29.133649598Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:09:29.978182291Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:09:30.57484107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:09:30.582921574Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:09:30.582942752Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:09:30.582966964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:09:30.589988343Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:09:31.254928101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:09:31.258417438Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:09:33.231083491Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-10T20:09:33.779118103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58546"
level=warn ts=2026-05-10T20:09:34.13355512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:09:35.291253402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:09:35.294742894Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:09:35.332691952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:09:35.340791363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:09:35.340815935Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:09:35.340855798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:09:35.346189416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:09:35.348397981Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:09:35.351720424Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:09:36.48400415Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=debug ts=2026-05-10T20:09:36.795709609Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:09:37.216576236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:09:39.13432661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:09:39.820307136Z caller=checkpoint.go:372 msg="writing series" size="70 MB"
level=debug ts=2026-05-10T20:09:39.820344115Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 MB" series=4
level=info ts=2026-05-10T20:09:40.772085291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:09:40.791109986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:09:42.989839421Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-10T20:09:43.03549961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:09:43.047091018Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-05-10T20:09:44.13366095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:09:45.220739596Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34538"
level=info ts=2026-05-10T20:09:45.256226948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:09:45.265098438Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:09:46.244005139Z caller=checkpoint.go:372 msg="writing series" size="516 kB"
level=info ts=2026-05-10T20:09:47.738966676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:09:47.753590327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:09:47.760934208Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:09:47.761155462Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:09:47.936173779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:09:47.953227961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:09:49.117068513Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:09:49.117600877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:09:49.133734018Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:09:49.133775832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:09:49.133787712Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:09:49.133807127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:09:49.33168464Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:09:49.496096366Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-10T20:09:50.372764551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:09:50.383563898Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:09:52.748970991Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-05-10T20:09:52.81418221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:09:52.824823502Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:09:53.949516735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:09:53.954381561Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:09:54.13444799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:09:56.002148447Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-05-10T20:09:59.133803579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:09:59.255439103Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T20:09:59.255461255Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-05-10T20:10:00.219232255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:10:00.227541794Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:10:01.151706747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:10:01.155391441Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:10:01.434822528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:10:01.438493941Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:10:01.531916247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:10:01.535391146Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:10:02.428227411Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:10:02.431809994Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:10:02.431836817Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:02.431861143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:10:02.434980409Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:10:02.508251956Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=warn ts=2026-05-10T20:10:04.133699402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:04.428681915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:10:04.433299267Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:10:05.760586535Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-10T20:10:05.762677183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:10:05.771878775Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:10:06.388379248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:10:06.393349996Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:10:06.672354261Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:10:09.013866415Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T20:10:09.133792061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:09.929100518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:10:09.933945379Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:10:12.267324674Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=warn ts=2026-05-10T20:10:14.134417219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:10:15.51983141Z caller=checkpoint.go:372 msg="writing series" size="43 kB"
level=info ts=2026-05-10T20:10:17.969012289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="13 MB" avg_comp="2.7 MB" total_uncomp="115 MB" avg_uncomp="23 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:10:17.979344697Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:10:18.024578804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:10:18.031310347Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:10:18.772499776Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=warn ts=2026-05-10T20:10:19.133571636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:19.13359584Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:10:19.133616683Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:10:19.133624572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:10:19.335756516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:10:21.609160038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:10:21.61298281Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:10:22.026078625Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T20:10:24.133832039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:24.419652539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:10:24.428401546Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:10:24.428424786Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:24.428448063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:10:24.431757524Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:10:25.314702989Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-10T20:10:25.314729114Z caller=checkpoint.go:384 msg="flushing series" totalSize="25 MB" series=8
level=info ts=2026-05-10T20:10:27.604001191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:10:27.608510167Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:10:27.608537474Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:27.608559193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:10:27.611706188Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:10:28.531602354Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T20:10:29.133558645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:10:31.784665149Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T20:10:32.626537768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:10:32.630507901Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-10T20:10:34.133690031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:10:35.038839862Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-05-10T20:10:35.645324122Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:10:37.116840642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:10:37.12184885Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:10:37.283540765Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:10:38.29081083Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=info ts=2026-05-10T20:10:38.669777446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:10:38.674419201Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:10:38.674499116Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:38.674553481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:10:38.677876562Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:10:39.134161745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:39.229458519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:10:39.233057845Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:10:39.448857407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="2.9 kB" avg_comp="2.9 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-05-10T20:10:39.452378448Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19e1332f52c%3A19e133397b5%3Acb5adb24?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-05-10T20:10:39.452399362Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:39.452425377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:10:39.455990599Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:10:40.848170267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:10:40.857413585Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:10:40.857440929Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:40.85748318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:10:40.865184631Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:10:41.408851041Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:10:41.41227011Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:10:41.544438228Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:10:42.482515078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:10:42.486019091Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:10:42.486043277Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:42.486082323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:10:42.488900552Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:10:44.134024437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:44.323019517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:10:44.331226982Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:10:44.686250305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:10:44.689968358Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:10:44.797511615Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:10:46.073220818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:10:46.076770716Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:10:46.076790863Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:46.076810733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:10:46.079693449Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-05-10T20:10:48.003738793Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47444"
level=debug ts=2026-05-10T20:10:48.053866732Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T20:10:48.05389278Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T20:10:49.117052622Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:10:49.117283518Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:10:49.133543919Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:10:49.133563516Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:10:49.133580365Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:10:49.133629116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:10:49.338990293Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:10:50.573318008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:10:50.577162508Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:10:50.577188887Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:50.577212064Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:10:50.580119237Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:10:51.09850035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:10:51.103600621Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:10:51.103622463Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:10:51.103661578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:10:51.107435007Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:10:51.302996578Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:10:51.468362697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:10:51.473907789Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:10:51.742742338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:10:51.752309721Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:10:54.134428223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:10:54.556628348Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=info ts=2026-05-10T20:10:55.752104028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:10:55.780047217Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:10:56.55643947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:10:56.560882875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:10:56.815217808Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:10:56.819198557Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:10:57.699646127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:10:57.708686694Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:10:57.80927547Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-05-10T20:10:58.011840742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:10:58.015221228Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:10:59.134281333Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:10:59.52231751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:10:59.530110724Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:11:01.062562751Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-10T20:11:01.253202103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:11:01.257750612Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:11:01.983485133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:11:01.988409859Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-05-10T20:11:03.787162415Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49580"
level=warn ts=2026-05-10T20:11:04.134165384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:11:04.319065067Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:11:04.319088601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-05-10T20:11:06.429054816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:11:06.4336896Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:11:07.568401107Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=warn ts=2026-05-10T20:11:09.133419332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:10.107767132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:11:10.115162766Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:11:10.372013065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:11:10.375745125Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:11:10.828901403Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:11:10.828925985Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=debug ts=2026-05-10T20:11:11.223781726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:11:14.073732114Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-10T20:11:14.134275514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:14.928566202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:11:14.943262254Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-05-10T20:11:15.228065927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:34866"
level=info ts=2026-05-10T20:11:15.774817019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:11:15.780333135Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:11:15.891695138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:11:15.895744158Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:11:17.328366242Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:11:17.756948218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:11:17.760613235Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:11:18.243182956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:11:18.246913371Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:11:18.477982417Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:11:18.479824674Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T20:11:18.628410706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:11:18.637472965Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:11:19.13375344Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:11:19.13378896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:11:19.133798013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:11:19.133842793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:11:19.342024174Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:11:20.483370097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:11:20.487227475Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:11:20.580615356Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-05-10T20:11:23.833742559Z caller=checkpoint.go:372 msg="writing series" size="102 kB"
level=warn ts=2026-05-10T20:11:24.133665453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:25.169208821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:11:25.172951275Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:11:26.667013359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:11:26.672902968Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:11:27.088039626Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-05-10T20:11:29.134250492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:11:30.339062577Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=info ts=2026-05-10T20:11:31.857035799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:11:31.862307208Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:11:33.592482678Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-05-10T20:11:34.134382082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:35.701029524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:11:35.705275452Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-10T20:11:36.84603422Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-05-10T20:11:36.84606633Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=8
level=info ts=2026-05-10T20:11:38.013348173Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:11:38.017664865Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:11:39.134232405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:11:40.099279089Z caller=checkpoint.go:372 msg="writing series" size="995 kB"
level=info ts=2026-05-10T20:11:40.553116498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:11:40.561699531Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:11:40.784325432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:11:40.79246572Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:11:41.844854541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:11:41.848722033Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:11:42.584920924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:11:42.589718815Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:11:42.817613339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:11:42.817689978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="443 B" avg_comp="443 B" total_uncomp="41 kB" avg_uncomp="41 kB" idle=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:11:42.821454732Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:11:43.35168407Z caller=checkpoint.go:372 msg="writing series" size="599 kB"
level=debug ts=2026-05-10T20:11:43.35170965Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=warn ts=2026-05-10T20:11:44.133910802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:45.770920668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:11:45.775990048Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:11:46.604802044Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-05-10T20:11:48.891319039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:11:48.898926049Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:11:49.117133351Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:11:49.117162966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:11:49.133429067Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:11:49.133457151Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:11:49.133466539Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:11:49.133531392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:11:49.344570534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:11:49.857870268Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-10T20:11:49.873828445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:11:49.87737055Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:11:53.110325111Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=debug ts=2026-05-10T20:11:54.045290895Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:11:54.133837973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:11:56.235831721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:11:56.241995967Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-05-10T20:11:56.363357262Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-10T20:11:56.395438249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:11:56.398911435Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:11:57.291863084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:11:57.301103364Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:11:57.428676021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:11:57.432299714Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:11:59.133583204Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:11:59.61670934Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:12:02.215167554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:12:02.218743331Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:12:02.87458363Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T20:12:02.874609752Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=6
level=warn ts=2026-05-10T20:12:04.133776075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:06.122839926Z caller=checkpoint.go:372 msg="writing series" size="379 kB"
level=info ts=2026-05-10T20:12:08.073896426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:12:08.08268494Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T20:12:09.134345998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:09.37542653Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-05-10T20:12:10.948176523Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:12:11.66869204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:12:11.672612227Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:12:12.141461581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:12:12.145029674Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:12:12.628628038Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=debug ts=2026-05-10T20:12:12.628689562Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T20:12:13.388553866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:12:13.392418261Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:12:13.706255638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:12:13.709812154Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:12:14.134160664Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:15.881258098Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=info ts=2026-05-10T20:12:16.475978685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:12:16.47954129Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-05-10T20:12:18.009691815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:43378"
level=info ts=2026-05-10T20:12:19.133315861Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:12:19.133349959Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:12:19.133358803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:12:19.133360077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:19.1343525Z caller=checkpoint.go:384 msg="flushing series" totalSize="259 kB" series=1
level=info ts=2026-05-10T20:12:19.14056895Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-05-10T20:12:19.142342989Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.00855964s
ts=2026-05-10T20:12:19.347237066Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:12:23.733067188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:12:23.736709664Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=warn ts=2026-05-10T20:12:24.13387093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:12:24.443653125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:12:24.453078635Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:12:25.00383215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:12:25.007288818Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:12:25.152828015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:12:25.156915338Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:12:25.237134258Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:12:25.240562581Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:12:25.950561189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:12:25.955450626Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:12:28.897357243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:12:28.906930509Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-05-10T20:12:29.133707334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-05-10T20:12:34.133558922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:35.458552527Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:12:35.458556643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:12:35.462917086Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:12:36.252172437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:12:36.255495448Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:12:37.640094745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:12:37.644043945Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T20:12:39.133479773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:12:41.436949026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:12:41.441231824Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:12:41.441254893Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:12:41.441280403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:12:41.444916658Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:12:41.60634027Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:12:41.61122679Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-10T20:12:44.133883376Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:12:45.475731133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:12:45.48276454Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:12:45.603250214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:12:45.609337063Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:12:47.292230017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:12:47.295904704Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:12:47.932918373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:12:47.940803189Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:12:49.117067122Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:12:49.117511281Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:12:49.133982353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:12:49.13401646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:12:49.13402545Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T20:12:49.134026863Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-05-10T20:12:49.134056358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:12:49.134189923Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-10T20:12:49.134198505Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T20:12:49.134218378Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.869µs success=true
level=debug ts=2026-05-10T20:12:49.134588617Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T20:12:49.350364829Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:12:51.385553688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:12:51.389072626Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:12:51.589259809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:12:52.387592353Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T20:12:53.08859098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:12:53.093408175Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-05-10T20:12:54.134230727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:12:55.642532597Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:12:55.642555758Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-10T20:12:57.668228644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:12:57.671994535Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:12:58.086213725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:12:58.093821798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:12:58.894243586Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:12:59.134025779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:13:00.204359413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:13:00.208980002Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:13:00.662069693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:13:00.665766323Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:13:02.14713685Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-05-10T20:13:03.760094874Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:13:04.134191385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:05.40113433Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-10T20:13:05.401157451Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-05-10T20:13:05.411016922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:13:05.415231844Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:13:06.345231399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:13:06.349295675Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:13:07.695175419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:13:07.70054556Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:13:08.652696555Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-10T20:13:09.134371124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:13:09.737693588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:13:09.745124802Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:13:11.906582635Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-10T20:13:11.974140829Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:13:11.977765454Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:13:14.134287708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:15.159668601Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-10T20:13:15.452263156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:13:15.455981212Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:13:15.4560039Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:13:15.4560267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:13:15.45948249Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:13:15.507681276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:13:15.511242521Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:13:18.411548075Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:13:19.133335108Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:13:19.133382329Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:13:19.133390892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:13:19.133389234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:13:19.35344646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:13:19.636405089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:13:19.641260244Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:13:20.780761438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:13:20.78449868Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:13:21.665301807Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T20:13:22.513961587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:13:22.518170952Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:13:23.496804522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:13:23.500567905Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:13:24.13351101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:24.917744184Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-10T20:13:27.618220818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=8 total_comp="21 MB" avg_comp="2.6 MB" total_uncomp="137 MB" avg_uncomp="17 MB" full=7 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:13:27.626529589Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:13:28.171742619Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=debug ts=2026-05-10T20:13:28.171764876Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-05-10T20:13:28.713186726Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:13:29.133580762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:31.423902127Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=info ts=2026-05-10T20:13:32.531754459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:13:32.537705171Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:13:32.668084228Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:13:32.986899479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:13:32.990609381Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-10T20:13:33.797506021Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:42362"
level=warn ts=2026-05-10T20:13:34.134234729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:34.676868293Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-10T20:13:35.259879184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:13:35.26635139Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-05-10T20:13:36.891663097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:13:36.896462964Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:13:37.929979913Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T20:13:38.561810116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:13:38.569723052Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T20:13:39.133411326Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:41.182673733Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=info ts=2026-05-10T20:13:43.550904069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:13:43.558668317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:13:44.133559965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:44.436627541Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=debug ts=2026-05-10T20:13:47.689904339Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-05-10T20:13:48.079954823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:13:48.084122401Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:13:49.116980549Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:13:49.118309198Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:13:49.133537058Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:13:49.133563306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:13:49.133679377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:13:49.133685651Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:13:49.356082979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:13:50.932423033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:13:50.93631243Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:13:50.942405855Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-10T20:13:51.304483709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:13:51.312181132Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:13:53.088028309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:13:53.095958179Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:13:53.753096499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:13:53.757044583Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:13:54.134176122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:13:54.196141308Z caller=checkpoint.go:372 msg="writing series" size="531 kB"
level=info ts=2026-05-10T20:13:55.233188496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:13:55.236569024Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:13:55.875413236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:13:55.879139274Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:13:56.224438243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:13:56.230101171Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:13:56.230130596Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:13:56.230154447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:13:56.234075907Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:13:57.449115999Z caller=checkpoint.go:372 msg="writing series" size="720 kB"
level=debug ts=2026-05-10T20:13:57.449138905Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=9
level=warn ts=2026-05-10T20:13:59.133734105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:00.064563848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:14:00.068498536Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:14:00.701320177Z caller=checkpoint.go:372 msg="writing series" size="237 kB"
level=info ts=2026-05-10T20:14:02.17309025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:14:02.176702223Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:14:03.64813081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:14:03.653405858Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:14:03.954544049Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-05-10T20:14:03.960532931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:14:03.964108235Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=warn ts=2026-05-10T20:14:04.133686114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:04.534085629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:14:04.538199267Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-10T20:14:07.208122893Z caller=checkpoint.go:372 msg="writing series" size="477 kB"
level=debug ts=2026-05-10T20:14:07.4954929Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:14:07.495497057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="353 kB" avg_comp="353 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:14:07.500477579Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:14:07.796410678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="353 kB" avg_comp="353 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" idle=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:14:07.806101065Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=warn ts=2026-05-10T20:14:09.134052888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:14:10.460873855Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-05-10T20:14:10.460900371Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-05-10T20:14:13.731152111Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-05-10T20:14:13.731181962Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-05-10T20:14:13.8836833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:14:13.891164576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:14:13.932497464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:14:13.937289227Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-10T20:14:14.134080691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:14.152219843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:14:14.155788167Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-10T20:14:15.238559546Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47572"
level=info ts=2026-05-10T20:14:16.151909576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:14:16.15664973Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:14:16.969737314Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-05-10T20:14:16.969760824Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=1
ts=2026-05-10T20:14:18.017855615Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56694"
level=info ts=2026-05-10T20:14:18.732767318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:14:18.737422823Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:14:19.134067273Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:14:19.134097184Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:14:19.134106966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:14:19.134170805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:14:19.359040316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:14:20.233072391Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T20:14:20.233100454Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-10T20:14:20.613934066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:14:20.618019511Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:14:21.55198176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:14:21.556587772Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:14:22.373104732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:14:22.377568895Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:14:23.472865784Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T20:14:24.133668373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:14:24.601161132Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:14:24.979376223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="16 MB" avg_uncomp="16 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:14:24.987342591Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:14:26.44674488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:14:26.450409492Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:14:26.725489664Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T20:14:26.725512011Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=warn ts=2026-05-10T20:14:29.13347007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:29.800031339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:14:29.803746038Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:14:29.978371228Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:14:31.0964835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:14:31.102856477Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19e1332ff1e%3A19e13687363%3A66e7d29?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-05-10T20:14:31.102880413Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:14:31.102906568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:14:31.10655425Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:14:33.230849309Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-10T20:14:33.80350808Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:50672"
level=warn ts=2026-05-10T20:14:34.133911294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:34.390658414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:14:34.39708687Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:14:35.957003065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:14:35.960727866Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:14:35.96074938Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:14:35.960772992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:14:35.963737809Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:14:36.054141712Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:14:36.484509244Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-05-10T20:14:36.987407568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:14:36.995920574Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T20:14:39.133790098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:14:39.820468802Z caller=checkpoint.go:372 msg="writing series" size="70 MB"
level=debug ts=2026-05-10T20:14:39.820507883Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 MB" series=4
level=info ts=2026-05-10T20:14:41.888496834Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:14:41.899937528Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:14:42.612346819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:14:42.623353716Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:14:42.677859751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:14:42.686741571Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:14:42.990450641Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-05-10T20:14:43.671890897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:14:43.689476492Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:14:43.721501461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:14:43.733062775Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-10T20:14:44.133670139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:45.869888362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:14:45.885687216Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:14:46.203811549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:14:46.218879296Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=debug ts=2026-05-10T20:14:46.244025774Z caller=checkpoint.go:372 msg="writing series" size="535 kB"
ts=2026-05-10T20:14:48.020898276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:34208"
level=info ts=2026-05-10T20:14:48.828265629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:14:48.843276711Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:14:49.118677016Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:14:49.12012213Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:14:49.133820903Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:14:49.133861234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:14:49.133873117Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:14:49.133885273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:14:49.362389928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:14:49.496570285Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-10T20:14:50.983391578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:14:50.995761053Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:14:52.387211768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:14:52.396429964Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:14:52.749427614Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-05-10T20:14:54.133723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:14:55.628063032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:14:55.634321335Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:14:56.002422053Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-10T20:14:58.769916106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:14:58.77428997Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=warn ts=2026-05-10T20:14:59.133887965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:14:59.255523641Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T20:14:59.255549833Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-10T20:15:00.400977439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:15:00.404601158Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:15:00.96138049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:15:00.965163781Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:15:01.112682776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:15:01.116349692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:15:02.508819141Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=debug ts=2026-05-10T20:15:03.254322126Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:15:03.254372823Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="353 kB" avg_comp="353 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" idle=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:15:03.264820184Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=warn ts=2026-05-10T20:15:04.133430306Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:05.240816631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:15:05.246277033Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:15:05.761927929Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-10T20:15:06.432946881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:15:06.4376066Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:15:09.014603829Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T20:15:09.133573927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:11.0634755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:15:11.067951194Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:15:12.268315206Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T20:15:12.706157738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:15:12.709802386Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:15:13.040770501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:15:13.048951158Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:15:13.048973106Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:15:13.049008563Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:15:13.056683742Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:15:13.297372825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:15:13.30106178Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:15:14.133810634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:15:15.243272679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:47192"
level=debug ts=2026-05-10T20:15:15.520856664Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:15:16.257461815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:15:16.261770839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
ts=2026-05-10T20:15:18.025060405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:57780"
level=debug ts=2026-05-10T20:15:18.167928778Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:15:18.169520473Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T20:15:18.773399394Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T20:15:19.133750623Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:15:19.133785823Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:15:19.133796523Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:15:19.133791695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:15:19.367006417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:15:22.026645962Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:15:23.136233648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:15:23.142998774Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:15:24.133940513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:24.822409449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:15:24.826186239Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:15:24.826213331Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:15:24.826237918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:15:24.829119151Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:15:25.023772201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:15:25.031847217Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:15:25.311604832Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-10T20:15:25.311630493Z caller=checkpoint.go:384 msg="flushing series" totalSize="25 MB" series=8
level=info ts=2026-05-10T20:15:25.727563939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:15:25.740988327Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:15:27.689105335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:15:27.694671932Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:15:28.53260994Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T20:15:29.133476495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:30.360829092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:15:30.366196417Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:15:30.94388234Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:15:30.94744329Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:15:31.786841957Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T20:15:32.15248922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:15:32.161368351Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:15:32.161389519Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:15:32.161419906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:15:32.171405456Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
ts=2026-05-10T20:15:33.80796659Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56472"
level=warn ts=2026-05-10T20:15:34.133536377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:34.328678651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:15:34.332077436Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:15:35.041908843Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=debug ts=2026-05-10T20:15:35.132414336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:15:36.067781222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:15:36.073603318Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:15:37.592602962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:15:37.596281254Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:15:38.218222822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:15:38.222577586Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:15:38.291820761Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-05-10T20:15:38.867585454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:15:38.870988989Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:15:39.133723001Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:15:40.378945939Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:15:40.383119801Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:15:40.770367723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:15:40.77947193Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:15:41.361084596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:15:41.365445184Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=debug ts=2026-05-10T20:15:41.54433777Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:15:41.562386659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:15:41.566068973Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:15:42.955839709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:15:42.960055656Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:15:44.051533229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:15:44.056344684Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=warn ts=2026-05-10T20:15:44.134388386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:15:44.798143534Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:15:46.107999613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:15:46.116186071Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:15:47.128466284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:15:47.132255683Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:15:47.228514835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:15:47.238035343Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
ts=2026-05-10T20:15:48.028231312Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:59666"
level=debug ts=2026-05-10T20:15:48.051930645Z caller=checkpoint.go:372 msg="writing series" size="613 kB"
level=debug ts=2026-05-10T20:15:48.051960891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T20:15:48.209671753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:15:48.214074626Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:15:48.521010775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:15:48.525685158Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:15:49.11703258Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:15:49.117465605Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=debug ts=2026-05-10T20:15:49.118808261Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-05-10T20:15:49.118859179Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1976048 n=1
level=debug ts=2026-05-10T20:15:49.118902297Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1976048 n=1
level=info ts=2026-05-10T20:15:49.13394365Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:15:49.133996969Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:15:49.134009446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:15:49.134005458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:15:49.369580837Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:15:51.303913516Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:15:53.663388376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:15:53.667965909Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-10T20:15:54.133806951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:15:54.556196187Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=debug ts=2026-05-10T20:15:56.788913384Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:15:57.809332757Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-05-10T20:15:59.133535275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:00.153976071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:16:00.158407314Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:16:01.062936955Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-10T20:16:03.901391614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:16:03.905075452Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:16:04.133508955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:16:04.316779023Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-10T20:16:04.316803063Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-10T20:16:06.280411646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:16:06.285135661Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:16:07.568807103Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:16:08.590565435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:16:08.599844109Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:16:09.133410507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:10.102434637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:16:10.105961769Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:16:10.731410746Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:16:10.826864845Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:16:10.826892887Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-10T20:16:12.240781768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:16:12.266710973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:16:12.266760318Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:16:12.266805097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:16:12.282084086Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:16:13.896785437Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:16:13.917158849Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:16:13.917200236Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:16:13.917239613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:16:13.930018734Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:16:14.074911541Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-10T20:16:14.133994801Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:15.324978203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:16:15.32894233Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:16:17.1715667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:16:17.175090908Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:16:17.328049446Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-10T20:16:18.03080857Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:54154"
level=info ts=2026-05-10T20:16:19.134058712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:16:19.134106677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:16:19.134116047Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:16:19.134107926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:16:19.37274326Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:16:20.580930538Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-10T20:16:21.568960935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:16:21.577367425Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:16:22.118118878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:16:22.124256126Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:16:22.397094984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:16:22.400551605Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:16:23.834202172Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-05-10T20:16:23.941509874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:16:23.946017177Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:16:24.02720384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:16:24.030803389Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:16:24.134045952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:25.686221739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:16:25.69383809Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:16:26.701020894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:16:26.704986417Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:16:27.089215155Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=info ts=2026-05-10T20:16:28.34299295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:16:28.350886357Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:16:29.133499531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:29.583004591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:16:29.587935586Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:16:30.340456504Z caller=checkpoint.go:372 msg="writing series" size="293 kB"
level=info ts=2026-05-10T20:16:31.242597513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:16:31.24614685Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:16:32.105031751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:16:32.109487879Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:16:32.782551307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:16:32.786735095Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=debug ts=2026-05-10T20:16:33.593503033Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=debug ts=2026-05-10T20:16:33.593523204Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-05-10T20:16:34.134113235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:35.770147948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:16:35.775341109Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:16:36.846361465Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-10T20:16:37.592220722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:16:37.596953254Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:16:37.847764187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:16:37.851464559Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:16:39.133422487Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:16:39.70535321Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:16:40.099584145Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-10T20:16:40.099606795Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-10T20:16:41.143354569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:16:41.147166016Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:16:43.352197418Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=warn ts=2026-05-10T20:16:44.133901832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:16:44.584387342Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T20:16:45.249850063Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:53614"
level=debug ts=2026-05-10T20:16:46.604584221Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=info ts=2026-05-10T20:16:46.886120134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:16:46.896903238Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:16:47.594889309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:16:47.599945405Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:16:49.117181808Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:16:49.117201559Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:16:49.133507543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:16:49.133546651Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:16:49.13355546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:16:49.133551152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:16:49.376080754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:16:49.858302345Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-05-10T20:16:50.007498772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:16:50.016335529Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:16:50.633244579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:16:50.638950315Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:16:53.111657973Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-10T20:16:53.863662017Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:16:53.871827001Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:16:54.134126097Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:55.237938789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:16:55.2417529Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:16:55.24177868Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:16:55.241804151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:16:55.244978716Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:16:56.363467059Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-10T20:16:56.959567729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:16:56.967315936Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:16:59.133443375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:16:59.581978799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:16:59.587383626Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:16:59.587410382Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:16:59.587434435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:16:59.591050745Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:16:59.617300284Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:17:00.129008627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:17:00.134061448Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:17:02.873374261Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T20:17:02.873404919Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.8 MB" series=7
level=info ts=2026-05-10T20:17:03.263220007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:17:03.266931137Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-05-10T20:17:03.814726968Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:49424"
level=warn ts=2026-05-10T20:17:04.134033829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:06.02221585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:17:06.027351405Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:17:06.12348556Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-05-10T20:17:07.141907915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:17:07.147772762Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:17:09.134163772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:17:09.376477172Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-05-10T20:17:09.677951387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:17:09.681570642Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:17:09.681596081Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:09.681623013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:17:09.684460561Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:17:09.74985323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:17:09.753265764Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:17:09.753287672Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:09.786357506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:17:09.789803695Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:17:09.982365375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:17:09.986978745Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:17:12.629534634Z caller=checkpoint.go:372 msg="writing series" size="552 kB"
level=info ts=2026-05-10T20:17:13.135843657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:17:13.139466791Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:17:13.139502445Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:13.139533886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:17:13.142662437Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:17:14.133655066Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:17:15.313549988Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:17:15.544542963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:17:15.882667045Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T20:17:15.882696891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T20:17:16.153486773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:17:16.162214971Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:17:16.162241712Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:16.162281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:17:16.172528071Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:17:16.529755032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:17:16.53420254Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:17:16.534229421Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:16.534251654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:17:16.537355437Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:17:17.463567508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:17:17.467868674Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
ts=2026-05-10T20:17:18.03503455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:56294"
level=info ts=2026-05-10T20:17:19.133829564Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:17:19.133862526Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:17:19.133871546Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:17:19.133913373Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:19.140555651Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-05-10T20:17:19.14320953Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008843833s
level=info ts=2026-05-10T20:17:19.281701072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:17:19.285989304Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:17:19.286018274Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:19.286044864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:17:19.289595155Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-10T20:17:19.379223687Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:17:19.477587692Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:17:19.481224531Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:17:20.710914155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:17:20.715092549Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=info ts=2026-05-10T20:17:21.111162853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:17:21.115409152Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:17:23.682187363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:17:23.686273576Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=warn ts=2026-05-10T20:17:24.134195985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:26.094914598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:17:26.098621165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:17:26.460401396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:17:26.46458133Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:17:27.51853482Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:17:27.522236149Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:17:28.912104844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:17:28.915838664Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:17:28.915864144Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:28.915894107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:17:28.918913134Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:17:29.124902229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:17:29.128495712Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:17:29.13358681Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:29.439373573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:17:29.444070297Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:17:31.200760069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:17:31.206012809Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:17:31.206039387Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:17:31.206067002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:17:31.210024069Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:17:31.220129139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:17:31.227956689Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:17:33.232488704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:17:33.235867035Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-05-10T20:17:33.817407421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33232"
level=warn ts=2026-05-10T20:17:34.133975989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:35.910595616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:17:35.91495201Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:17:37.018549413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:17:37.02285507Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:17:39.134208132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:39.436185151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:17:39.447202373Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:17:41.568418683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:17:41.576801754Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:17:43.464194793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:17:43.46825768Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:17:44.112573983Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:17:44.133705566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:44.532448179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:17:44.540536638Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:17:47.237213746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:17:47.240965766Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:17:47.762142549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:17:47.769364926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-05-10T20:17:48.038219519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:33064"
level=info ts=2026-05-10T20:17:48.314210651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:17:48.318914393Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:17:49.117094992Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:17:49.117105929Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:17:49.1334177Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T20:17:49.133451025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:17:49.133458458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:17:49.133471364Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:17:49.133481687Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T20:17:49.133570661Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-10T20:17:49.133577836Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T20:17:49.13359696Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.767µs success=true
level=debug ts=2026-05-10T20:17:49.134140313Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-05-10T20:17:49.196831827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:17:49.201529285Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-10T20:17:49.381500531Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:17:50.650212514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:17:50.653769523Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:17:52.387211437Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=info ts=2026-05-10T20:17:53.403481314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:17:53.406943132Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:17:54.133495244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:17:54.180695843Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:17:55.641916456Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:17:55.641939621Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-10T20:17:58.37599491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:17:58.379827542Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:17:58.893239547Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:17:59.133401663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:17:59.531543948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:17:59.535171225Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:17:59.876568963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:17:59.885047704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:18:00.395761228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:18:00.400110115Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:18:01.550519929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:18:01.555301516Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:18:02.147064164Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-05-10T20:18:02.494724128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:18:02.499608512Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-10T20:18:04.133699108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:18:04.351939799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:18:04.356476201Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:18:05.40233484Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T20:18:05.40236074Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-05-10T20:18:06.113917403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:18:06.118428522Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:18:06.310914075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:18:06.314727755Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:18:08.156690526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:18:08.161085755Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:18:08.65257346Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-05-10T20:18:09.13440636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:11.905308114Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-10T20:18:13.917485576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:18:13.923071169Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-10T20:18:14.134269873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:15.158255288Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=info ts=2026-05-10T20:18:15.791860468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:18:15.79580403Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:18:18.411815646Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:18:18.548304814Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:18:18.553211437Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:18:18.660437839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:18:18.664104101Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:18:18.664128956Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:18:18.664155256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:18:18.667740348Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:18:18.945316629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:18:19.134067197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:18:19.134106509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:18:19.134121243Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:18:19.134151537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:18:19.384284976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:18:19.797010541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:18:19.801685177Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=info ts=2026-05-10T20:18:21.640602632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:18:21.644471297Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:18:21.664747819Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T20:18:21.776165349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:18:21.77970108Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:18:24.133557355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:24.917396212Z caller=checkpoint.go:372 msg="writing series" size="9.5 kB"
level=info ts=2026-05-10T20:18:27.472178195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:18:27.481828077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:18:28.171590282Z caller=checkpoint.go:372 msg="writing series" size="679 kB"
level=debug ts=2026-05-10T20:18:28.17161758Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-05-10T20:18:28.578008713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:18:28.579666262Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-05-10T20:18:28.869505447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:18:28.876744959Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:18:29.133904683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:31.424140774Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-05-10T20:18:32.701626792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:18:32.705422667Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
ts=2026-05-10T20:18:33.822593837Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:39252"
level=warn ts=2026-05-10T20:18:34.134252117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:34.676448444Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-10T20:18:36.427778769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:18:36.432505238Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:18:37.452695377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:18:37.459849321Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:18:37.815964635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:18:37.82325273Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:18:37.9296847Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=warn ts=2026-05-10T20:18:39.13354879Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:18:39.195687142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:18:39.199384681Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:18:39.19940746Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:18:39.199432238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:18:39.203563094Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:18:40.755905293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:18:40.759422108Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:18:41.182703736Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=info ts=2026-05-10T20:18:42.006602848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:18:42.013994615Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T20:18:44.1338346Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:44.435562474Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-10T20:18:44.5442109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:18:44.549975755Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
ts=2026-05-10T20:18:45.256579582Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:45736"
level=info ts=2026-05-10T20:18:45.613186767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:18:45.616778983Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:18:46.131493438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:18:46.136240212Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:18:46.345826451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:18:46.350042571Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:18:47.323538706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:18:47.327879224Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:18:47.688875733Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=info ts=2026-05-10T20:18:49.116985394Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:18:49.11699103Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:18:49.1333181Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:18:49.133348347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:18:49.13335922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:18:49.133404579Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:18:49.15152367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:18:49.155049342Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-10T20:18:49.386929786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:18:50.063742293Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:18:50.343776986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:18:50.347575322Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:18:50.941363287Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-10T20:18:52.760964084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:18:52.764575525Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:18:54.134018718Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:18:54.194457596Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=info ts=2026-05-10T20:18:54.347350594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:18:54.351651282Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:18:55.164050567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:18:55.168203551Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:18:56.979600475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:18:56.983875284Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:18:57.122840108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:18:57.126511878Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:18:57.447787457Z caller=checkpoint.go:372 msg="writing series" size="582 kB"
level=debug ts=2026-05-10T20:18:57.447810327Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-05-10T20:18:59.134274793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:00.700940287Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-10T20:19:01.660521374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:19:01.664294952Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:19:01.959987373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:19:01.964084958Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
ts=2026-05-10T20:19:03.825828276Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:58406"
level=debug ts=2026-05-10T20:19:03.953512641Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-05-10T20:19:04.134080563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:05.280396544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:19:05.284932558Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:19:06.15375707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:19:06.157443537Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:19:07.207322698Z caller=checkpoint.go:372 msg="writing series" size="608 kB"
level=debug ts=2026-05-10T20:19:07.207345782Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T20:19:08.731562599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:19:08.735259487Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T20:19:09.133623061Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:10.460108727Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:19:12.568998896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:19:12.573081674Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:19:13.732289075Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-10T20:19:13.732317862Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=2
level=warn ts=2026-05-10T20:19:14.133907755Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:15.828123999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:19:15.837234753Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:19:16.968527235Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-05-10T20:19:16.968552878Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=1
level=info ts=2026-05-10T20:19:17.678646677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:19:17.682743693Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:19:17.844358498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T20:19:18.045062348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:38862"
level=debug ts=2026-05-10T20:19:18.786790216Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:19:19.133321491Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:19:19.133351506Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:19:19.133360567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:19:19.133378184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:19:19.389570349Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:19:20.231402079Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T20:19:20.231424675Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-10T20:19:22.434967224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:19:22.442384265Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:19:23.066114549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:19:23.074186474Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:19:23.472469639Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T20:19:24.134033554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:25.863878113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:19:25.87080719Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:19:26.087454468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:19:26.091796908Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:19:26.091820293Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:19:26.091846565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:19:26.095808647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:19:26.725252837Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T20:19:26.725280106Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T20:19:29.017023919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:19:29.022780332Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-05-10T20:19:29.134284425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:29.977918242Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:19:30.077024813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:19:30.081752623Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:19:30.610091735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:19:30.617816089Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:19:31.128528227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:19:31.132961359Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:19:32.158929868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:19:32.164004267Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:19:32.746533967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:19:32.750038797Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:19:33.23125215Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-05-10T20:19:33.828863669Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:46802"
level=warn ts=2026-05-10T20:19:34.133984178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:35.589499627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:19:35.593243518Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:19:36.483843958Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-05-10T20:19:38.779534711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:19:38.784282521Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:19:38.953823431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:19:38.957419559Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:19:39.133379906Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:39.172535161Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:19:39.176845992Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:19:39.838176023Z caller=checkpoint.go:372 msg="writing series" size="70 MB"
level=debug ts=2026-05-10T20:19:39.838207319Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 MB" series=4
level=info ts=2026-05-10T20:19:42.095936238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:19:42.112664282Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:19:42.989878494Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-10T20:19:44.133778381Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:19:46.23259042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:19:46.243496059Z caller=checkpoint.go:372 msg="writing series" size="547 kB"
level=error ts=2026-05-10T20:19:46.250861132Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:19:48.153945229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:19:48.168589868Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:19:49.116982731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:19:49.117630275Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:19:49.133835562Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:19:49.133873077Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:19:49.133881982Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:19:49.13389695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:49.387438115Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-05-10T20:19:49.392875055Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:19:49.496016474Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-05-10T20:19:49.664941283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:19:49.67595752Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:19:49.906536721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:19:49.918630547Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:19:51.303476137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:19:51.314992572Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:19:51.651835696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:19:51.663253991Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:19:51.663301693Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:19:51.663344957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:19:51.675363063Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:19:52.748952532Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-05-10T20:19:54.134334441Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:56.00161802Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-10T20:19:56.048831249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:19:56.053405288Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=warn ts=2026-05-10T20:19:59.134016352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:19:59.255077521Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=debug ts=2026-05-10T20:19:59.255101781Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-05-10T20:19:59.709999826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:19:59.714024032Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:19:59.832235073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:19:59.836328969Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:20:01.410046743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:20:01.414429615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:20:02.507930287Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=debug ts=2026-05-10T20:20:02.949479697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:20:04.134248853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:04.645768564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:20:04.653910727Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:20:05.76121491Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=info ts=2026-05-10T20:20:07.374568435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:20:07.378283663Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:20:08.244856794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:20:08.252903148Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:20:09.01438141Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T20:20:09.133702313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:09.35093629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:20:09.358485139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:20:12.267820184Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T20:20:12.332761214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:20:12.337685884Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:20:14.134368354Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:14.694538116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:20:14.701956637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:20:15.519592797Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-05-10T20:20:15.990009632Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:20:15.990080654Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="585 kB" avg_comp="585 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" idle=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:20:16.00332106Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=info ts=2026-05-10T20:20:17.059115021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:20:17.067809083Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:20:17.110087837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:20:17.1136786Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:20:18.773121199Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T20:20:18.994759142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:20:18.99886573Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:20:18.998886594Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:20:18.998912226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:20:19.002379998Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:20:19.134058295Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:20:19.134091693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:20:19.134100648Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:20:19.134120313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:20:19.398503724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:20:19.907781987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:20:19.911495679Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:20:20.097828708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:20:20.104570217Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:20:20.104599714Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:20:20.104623214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:20:20.108544157Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:20:21.579987546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:20:21.583773316Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:20:22.026248321Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-05-10T20:20:24.134122525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:20:25.313490132Z caller=checkpoint.go:372 msg="writing series" size="24 MB"
level=debug ts=2026-05-10T20:20:25.313515837Z caller=checkpoint.go:384 msg="flushing series" totalSize="25 MB" series=8
level=info ts=2026-05-10T20:20:26.26137463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:20:26.265151386Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:20:26.496765185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:20:26.504454595Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-10T20:20:28.531766306Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-05-10T20:20:28.659030522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:20:28.662973606Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:20:29.134259635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:29.138956822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:20:29.142411355Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:20:31.784959568Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T20:20:32.678086989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:20:32.683927016Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=warn ts=2026-05-10T20:20:34.133494243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:34.230685248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:20:34.23412596Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:20:35.038868843Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-05-10T20:20:35.427207058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:20:35.430800483Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:20:37.287531395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:20:37.291568515Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:20:37.291593951Z caller=flush.go:240 component=ingester loop=9 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:20:37.291619087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:20:37.294884657Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:20:37.967728477Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:20:38.290841Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:20:38.290958443Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=error ts=2026-05-10T20:20:38.294866893Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:20:38.459443537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:20:38.464417411Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-05-10T20:20:39.134148778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:41.337548846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:20:41.342307311Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:20:41.544349521Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:20:42.311336659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:20:42.315556001Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:20:42.315581642Z caller=flush.go:240 component=ingester loop=24 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:20:42.315607175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:20:42.320605035Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:20:43.647573474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:20:43.651439973Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:20:44.13417042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:20:44.383000383Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:20:44.796835045Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:20:46.854142723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:20:46.858031767Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:20:48.050255855Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-10T20:20:48.050287079Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T20:20:49.11717711Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:20:49.117613408Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:20:49.124892311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:20:49.130127257Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:20:49.134243252Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:20:49.134264257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:20:49.134272281Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:20:49.134274428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:20:49.401624001Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:20:51.302958316Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:20:53.560109134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:20:53.565562Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=warn ts=2026-05-10T20:20:54.133713129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:20:54.526913382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:20:54.532052429Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:20:54.555843978Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-05-10T20:20:54.819531993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:20:54.824181535Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=info ts=2026-05-10T20:20:56.905248898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:20:56.914496319Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:20:57.457400602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:20:57.461914321Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:20:57.809081323Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-05-10T20:20:58.565290174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:20:58.570718649Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=warn ts=2026-05-10T20:20:59.1337496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:21:00.584339701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:21:00.592165855Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:21:01.062555496Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-05-10T20:21:03.836906967Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:38648"
level=warn ts=2026-05-10T20:21:04.134097943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:04.316765587Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-05-10T20:21:04.316802644Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-05-10T20:21:06.583377114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:21:06.588817785Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:21:07.078443032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:21:07.086733722Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:21:07.458167813Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:21:07.461741109Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:21:07.461764033Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:21:07.461785843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:21:07.464614466Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:21:07.543062853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:21:07.552252014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:21:07.568394026Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:21:08.47901318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:21:08.487136286Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:21:09.133972524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:21:09.503751614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:21:09.508489182Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:21:10.826037089Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:21:10.826070549Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=info ts=2026-05-10T20:21:11.199945243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:21:11.211074352Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:21:14.07549353Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-05-10T20:21:14.098160697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:21:14.102719421Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:21:14.133933309Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:14.244328368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:21:14.275314678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:21:14.279042952Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:21:15.550299632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:21:15.554082535Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:21:17.327303426Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:21:19.133670455Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:21:19.133704931Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:21:19.133713699Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:21:19.133708461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:21:19.404968775Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=debug ts=2026-05-10T20:21:20.58001275Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-10T20:21:23.300398885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:21:23.305693539Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:21:23.589569274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:21:23.593608936Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-10T20:21:23.834923916Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-05-10T20:21:24.133834297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:21:24.990065447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:21:24.994262339Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:21:25.086653443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:21:25.090234456Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:21:25.628087079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:21:25.631784017Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:21:25.918749081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:21:25.92196278Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:21:27.08685938Z caller=checkpoint.go:372 msg="writing series" size="352 kB"
level=info ts=2026-05-10T20:21:27.24246298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:21:27.246127828Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-10T20:21:29.13394464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:30.339606357Z caller=checkpoint.go:372 msg="writing series" size="348 kB"
level=info ts=2026-05-10T20:21:31.530772733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:21:31.535423513Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:21:32.645170448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:21:32.648846573Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:21:33.592538987Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=debug ts=2026-05-10T20:21:33.592560601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-05-10T20:21:33.832013824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:21:33.835387653Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:21:34.134499014Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:35.633825796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:21:36.811116691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:21:36.816162781Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:21:36.845220787Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=info ts=2026-05-10T20:21:38.909252485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=77747a4d97aab910 immediate=false num_chunks=1 total_comp="428 kB" avg_comp="428 kB" total_uncomp="1.6 MB" avg_uncomp="1.6 MB" full=1 labels="{stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:21:38.91419644Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/77747a4d97aab910/19e134316bd%3A19e1350679f%3A7ee71667?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-yige\"}"
level=error ts=2026-05-10T20:21:38.91422459Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:21:38.914251676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:21:38.920188422Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-10T20:21:38.979564357Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:21:39.134235319Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:40.0987883Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-05-10T20:21:40.098810764Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-05-10T20:21:41.111384656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:21:41.116416279Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:21:43.352040315Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=warn ts=2026-05-10T20:21:44.133687215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:46.604763567Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-05-10T20:21:47.471499946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:21:47.477336334Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-05-10T20:21:48.053095285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:49330"
level=info ts=2026-05-10T20:21:48.757569944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:21:48.765984454Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:21:48.766010204Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:21:48.766045819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:21:48.776151883Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:21:49.057387203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:21:49.062699864Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:21:49.116981626Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:21:49.117783758Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:21:49.134121549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:21:49.134158529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:49.134167992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:21:49.134180792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:21:49.407465597Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:21:49.857172708Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=info ts=2026-05-10T20:21:50.746119077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:21:50.751019867Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-10T20:21:51.621263882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:21:51.625897791Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=info ts=2026-05-10T20:21:51.728312962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:21:51.738511803Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:21:52.734896318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:21:52.73934823Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-05-10T20:21:53.111809223Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-05-10T20:21:54.133736714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:21:56.36314508Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-05-10T20:21:58.402329944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:21:58.408175422Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-10T20:21:59.134100321Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:21:59.264460335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:21:59.273757377Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:21:59.577297995Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:21:59.581027316Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:21:59.616337456Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:22:00.537092867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:22:00.540883131Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:22:00.897608207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:22:00.906524076Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:22:02.306113869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:22:02.315414728Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:22:02.315446123Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:22:02.315477203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:22:02.322524801Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:22:02.8763188Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-05-10T20:22:02.876352471Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=7
level=warn ts=2026-05-10T20:22:04.134453274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:04.732543293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:22:04.736422797Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:22:04.73644905Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:22:04.736475793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:22:04.739555879Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:22:06.12233036Z caller=checkpoint.go:372 msg="writing series" size="288 kB"
level=info ts=2026-05-10T20:22:06.288017631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:22:06.292702787Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:22:06.348488666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:22:07.673774644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:22:07.679291466Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:22:08.629453822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:22:08.638804151Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:22:08.740387339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:22:08.744520441Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-05-10T20:22:09.134098169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:22:09.377165451Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-05-10T20:22:11.980671969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:22:11.984276987Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:22:12.050420205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:22:12.054270425Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=debug ts=2026-05-10T20:22:12.630342457Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=info ts=2026-05-10T20:22:13.561335917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:22:13.571900132Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:22:14.134375803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:14.379220624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:22:14.385337993Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:22:15.221187951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:22:15.225568689Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-10T20:22:15.881579887Z caller=checkpoint.go:372 msg="writing series" size="259 kB"
level=debug ts=2026-05-10T20:22:15.881602967Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-05-10T20:22:17.115910368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:22:17.119726772Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:22:19.076754945Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:22:19.080282184Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:22:19.134117453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:22:19.134144817Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:22:19.134153078Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:22:19.134145935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:19.141857219Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-05-10T20:22:19.143540098Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009776047s
ts=2026-05-10T20:22:19.4110663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:22:19.856422999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:22:19.859983507Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-05-10T20:22:24.134105537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:27.631219557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:22:27.636238228Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:22:28.341313464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:22:28.345332196Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:22:29.096363634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:22:29.101813433Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:22:29.134247056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:22:33.260264809Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:22:33.262018436Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-05-10T20:22:33.512109285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-05-10T20:22:34.133967951Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:34.975776089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:22:34.979427915Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:22:36.37754458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:22:36.381983105Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:22:37.934092122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:22:37.94377735Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:22:39.134183175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:41.58525107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:22:41.58914733Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:22:44.133416242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:44.327109046Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:22:44.331364702Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:22:44.944911269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:22:44.952664782Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:22:44.952688433Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:22:44.952715818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:22:44.960002968Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:22:45.546800432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:22:45.551479897Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=info ts=2026-05-10T20:22:46.529957948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:22:46.533529284Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:22:46.566187295Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:22:46.570878214Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:22:47.539252092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:22:47.543037961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:22:48.426626661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:22:48.434959315Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:22:49.117001731Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:22:49.117218842Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:22:49.133507175Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-05-10T20:22:49.133496501Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:22:49.133555851Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-05-10T20:22:49.133549547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:49.13356865Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-05-10T20:22:49.133818756Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-05-10T20:22:49.133828262Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-05-10T20:22:49.133848701Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.765µs success=true
level=debug ts=2026-05-10T20:22:49.134368974Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
ts=2026-05-10T20:22:49.415965983Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:22:51.061893155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:22:51.065349863Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:22:51.819963616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:22:51.828832665Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:22:52.387520073Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=warn ts=2026-05-10T20:22:54.13403448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:22:55.050559213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:22:55.05909555Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:22:55.059120397Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:22:55.059148846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:22:55.066418654Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:22:55.641493232Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-05-10T20:22:55.641516281Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-05-10T20:22:55.648379132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:22:55.655350799Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:22:57.406587048Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:22:57.41039474Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:22:57.508599387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:22:57.512952355Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:22:57.615215587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:22:57.620602606Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-10T20:22:58.894090492Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-05-10T20:22:59.133809987Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:00.708326649Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:23:02.146617407Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=info ts=2026-05-10T20:23:02.460299209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:23:02.46951937Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:23:02.65507191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="646 B" avg_comp="646 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:23:02.6587013Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:23:02.658726009Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:23:02.658750586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:23:02.662039755Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:23:04.133989524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:05.285964305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:23:05.289660308Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:23:05.402076143Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-05-10T20:23:05.402098992Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=3
level=info ts=2026-05-10T20:23:07.558923664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:23:07.562842657Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:23:08.652909652Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-05-10T20:23:09.133668498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:09.286173815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:23:09.29079433Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:23:11.042989673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:23:11.047507423Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:23:11.906246078Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-05-10T20:23:12.686249309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:23:12.695475441Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-05-10T20:23:14.133922468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:15.158925023Z caller=checkpoint.go:372 msg="writing series" size="228 kB"
level=debug ts=2026-05-10T20:23:18.412241109Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:23:19.133849122Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:23:19.133911391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:23:19.133919895Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:23:19.133924405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:19.306278042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:23:19.310556425Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:23:19.344855123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:23:19.349122189Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-10T20:23:19.418803656Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:23:20.181288492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:23:20.184953343Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:23:21.666058869Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=info ts=2026-05-10T20:23:22.223992896Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:23:22.227739347Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:23:23.461232986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:23:23.465751626Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-10T20:23:24.133609838Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:24.917411847Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-05-10T20:23:26.764445042Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:23:27.77040098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:23:28.171157009Z caller=checkpoint.go:372 msg="writing series" size="605 kB"
level=debug ts=2026-05-10T20:23:28.171186301Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-10T20:23:29.018267671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:23:29.025020639Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-05-10T20:23:29.133492873Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:30.413441373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:23:30.421062593Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:23:31.424404783Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=info ts=2026-05-10T20:23:31.952844062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:23:31.956590541Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:23:32.62119053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:23:32.62643806Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:23:33.922506058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:23:33.929926844Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:23:33.929961464Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:23:33.930009045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:23:33.936762444Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T20:23:34.133387131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:34.676918072Z caller=checkpoint.go:372 msg="writing series" size="1.9 kB"
level=info ts=2026-05-10T20:23:36.100288028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:23:36.107833725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:23:36.827976487Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:23:36.832900935Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:23:37.813449514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:23:37.817189828Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:23:37.817211043Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:23:37.817239057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:23:37.820827667Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:23:37.930300184Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=info ts=2026-05-10T20:23:38.68261915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:23:38.687418401Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=warn ts=2026-05-10T20:23:39.134347234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:39.169509389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:23:39.17664559Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:23:40.489754519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:23:40.494611354Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:23:41.182797087Z caller=checkpoint.go:372 msg="writing series" size="9.2 kB"
level=info ts=2026-05-10T20:23:42.797759325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:23:42.801427337Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:23:42.801448396Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:23:42.801469859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:23:42.804359048Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:23:43.579552617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:23:43.583242752Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-05-10T20:23:44.134050392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:44.435928101Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-05-10T20:23:45.218178631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:23:45.222430017Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:23:46.834476491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:23:46.838567874Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:23:46.838591432Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:23:46.838618498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:23:46.841868247Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:23:47.508402003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:23:47.516186592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:23:47.688745286Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
ts=2026-05-10T20:23:48.061844338Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:47702"
level=info ts=2026-05-10T20:23:48.387419705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:23:48.391669387Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-10T20:23:49.117270644Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:23:49.117708368Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:23:49.13409426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:23:49.13411935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:23:49.13412924Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:23:49.13414834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:23:49.421512194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=info ts=2026-05-10T20:23:49.759906127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:23:49.763490679Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:23:50.044235029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:23:50.04775833Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:23:50.331745348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:23:50.335233644Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:23:50.941950945Z caller=checkpoint.go:372 msg="writing series" size="9.9 kB"
level=info ts=2026-05-10T20:23:52.569846942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:23:52.573374084Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-05-10T20:23:54.134157933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:23:54.195547157Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=debug ts=2026-05-10T20:23:56.308824507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:23:57.4481417Z caller=checkpoint.go:372 msg="writing series" size="660 kB"
level=debug ts=2026-05-10T20:23:57.448166572Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=9
level=warn ts=2026-05-10T20:23:59.133702783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:23:59.753041898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:23:59.758315041Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:24:00.700935862Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-05-10T20:24:01.206461497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:24:01.209912837Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:24:01.209932606Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:01.209955229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:24:01.212866506Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:24:01.24795247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:24:01.251565047Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:24:03.135418346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:24:03.141436057Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
ts=2026-05-10T20:24:03.850086876Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:57774"
level=debug ts=2026-05-10T20:24:03.953769783Z caller=checkpoint.go:372 msg="writing series" size="248 kB"
level=warn ts=2026-05-10T20:24:04.134159428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:05.732002355Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.5 kB" avg_comp="1.5 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:24:05.735931391Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:24:06.061829317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:24:06.065304717Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:24:06.065326554Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:06.065347454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:24:06.068281853Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:24:07.206671841Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=warn ts=2026-05-10T20:24:09.134243113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:09.50200848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:24:09.508854215Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:24:10.460071364Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-05-10T20:24:10.460093759Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=4
level=info ts=2026-05-10T20:24:13.007555442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:24:13.011786778Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:24:13.10289236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:24:13.110313622Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:24:13.733788298Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-05-10T20:24:13.733814164Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=1
level=warn ts=2026-05-10T20:24:14.134301458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:15.016564605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.0 kB" avg_comp="6.0 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:24:15.020090589Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:24:15.020112636Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:15.020135453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:24:15.023053119Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=info ts=2026-05-10T20:24:15.928837223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:24:15.932408003Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=info ts=2026-05-10T20:24:15.987484975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:24:15.991668533Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:24:15.991687147Z caller=flush.go:240 component=ingester loop=2 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:15.991709216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:24:15.995445377Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:24:16.047709906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:24:16.054724057Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-05-10T20:24:16.339778037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:24:16.34307363Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:24:16.969726567Z caller=checkpoint.go:372 msg="writing series" size="4.5 MB"
level=debug ts=2026-05-10T20:24:16.969749382Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=1
level=info ts=2026-05-10T20:24:17.492473651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:24:17.496813795Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-10T20:24:18.064947319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:52506"
level=info ts=2026-05-10T20:24:19.133382437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-05-10T20:24:19.133385122Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:24:19.133411177Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:24:19.133419689Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:24:19.424523268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=debug ts=2026-05-10T20:24:20.231732245Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-05-10T20:24:20.231758177Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-05-10T20:24:22.638126082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:24:22.642041807Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:24:23.473277698Z caller=checkpoint.go:372 msg="writing series" size="971 kB"
level=warn ts=2026-05-10T20:24:24.13400723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:24:24.713301717Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:24:26.306772711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:24:26.310724825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:24:26.72507288Z caller=checkpoint.go:372 msg="writing series" size="189 kB"
level=debug ts=2026-05-10T20:24:26.72509576Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-05-10T20:24:26.780474406Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:24:26.784740273Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:24:28.059203765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:24:28.067476517Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:24:29.062086633Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:24:29.066396924Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-05-10T20:24:29.133903188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:29.182019345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:24:29.185495085Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:24:29.213731441Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-05-10T20:24:29.977705655Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=info ts=2026-05-10T20:24:30.506243584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:24:30.509764555Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:24:30.700207008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:24:30.705096438Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:24:33.204780435Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:24:33.212288856Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=debug ts=2026-05-10T20:24:33.231445227Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-05-10T20:24:33.472561655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:24:33.480805368Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:24:33.890979807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:24:33.89550235Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:24:34.116220819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:24:34.119922525Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-05-10T20:24:34.134038864Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:34.321127753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:24:34.32521651Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:24:36.308328515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:24:36.311944062Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:24:36.484351053Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-05-10T20:24:38.162087464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:24:38.166029269Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=warn ts=2026-05-10T20:24:39.133842336Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:39.142009015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:24:39.145820104Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:24:39.891876641Z caller=checkpoint.go:372 msg="writing series" size="70 MB"
level=debug ts=2026-05-10T20:24:39.891912409Z caller=checkpoint.go:384 msg="flushing series" totalSize="70 MB" series=4
level=info ts=2026-05-10T20:24:40.64354585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:24:40.655272611Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:24:42.166270183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:24:42.177679546Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:24:42.990200688Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-05-10T20:24:44.133686149Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:24:45.279909198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:37292"
level=debug ts=2026-05-10T20:24:46.243811213Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=info ts=2026-05-10T20:24:49.116989046Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:24:49.117321162Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:24:49.133814363Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:24:49.133852927Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:24:49.133864872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:24:49.134043034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:24:49.427847555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:24:49.475517204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:24:49.494156262Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:24:49.49418491Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:49.494213281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:24:49.496157463Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=error ts=2026-05-10T20:24:49.502925338Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:24:52.749204011Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-05-10T20:24:54.133940488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:24:55.035222274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:24:55.039122619Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:24:55.039145721Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:24:55.039167686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:24:55.042040343Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:24:55.891533025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:24:55.895132142Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:24:55.988669949Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:24:55.988684969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e2f32baf25a756c3 immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="191 kB" avg_uncomp="191 kB" idle=1 labels="{stream=\"build-e2e-sspn\"}"
level=error ts=2026-05-10T20:24:55.993051043Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e2f32baf25a756c3/19e1343265d%3A19e135902ae%3A79c24ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-sspn\"}"
level=debug ts=2026-05-10T20:24:56.002144751Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=info ts=2026-05-10T20:24:58.959126467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:24:58.963773992Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-05-10T20:24:59.133603193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:24:59.254998015Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=info ts=2026-05-10T20:25:00.179517801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:25:00.187219273Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:25:00.720824398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:25:00.724735392Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:25:01.568357936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:25:01.572059585Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:25:02.50885353Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=debug ts=2026-05-10T20:25:02.50887593Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-05-10T20:25:03.119845992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:25:03.127830068Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=info ts=2026-05-10T20:25:03.712306541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:25:03.724220537Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-05-10T20:25:04.133815141Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:25:05.761209122Z caller=checkpoint.go:372 msg="writing series" size="4.3 kB"
level=debug ts=2026-05-10T20:25:09.014257491Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-05-10T20:25:09.134096972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:09.663221912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:25:09.667246449Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:25:10.31006831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e2f32baf25a756c3 immediate=false num_chunks=1 total_comp="55 kB" avg_comp="55 kB" total_uncomp="191 kB" avg_uncomp="191 kB" full=1 labels="{stream=\"build-e2e-sspn\"}"
level=error ts=2026-05-10T20:25:10.313768424Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e2f32baf25a756c3/19e1343265d%3A19e135902ae%3A79c24ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-sspn\"}"
level=info ts=2026-05-10T20:25:11.026761926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:25:11.030937209Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:25:12.2680626Z caller=checkpoint.go:372 msg="writing series" size="450 kB"
level=info ts=2026-05-10T20:25:12.479171001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:25:12.48267652Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-05-10T20:25:14.134110201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:14.376597822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:25:14.379828268Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:25:14.905772317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:25:14.909343072Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:25:14.90936758Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:25:14.909389314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:25:14.912253555Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:25:15.520133819Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:25:16.424075618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:25:16.428794206Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:25:16.989072184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:25:16.99330571Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
ts=2026-05-10T20:25:18.068517515Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:41608"
level=debug ts=2026-05-10T20:25:18.773216291Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-05-10T20:25:19.133718675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:25:19.133746718Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:25:19.133754581Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:25:19.133774927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:19.169888758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:25:19.173245691Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-05-10T20:25:19.431730233Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-xpqd9-ae8f5e53 10.128.0.66:7946"
level=info ts=2026-05-10T20:25:21.762465605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:25:21.771217161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:25:22.026367454Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-05-10T20:25:22.503306895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:25:22.511272019Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:25:22.767023864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:25:22.770458909Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T20:25:24.133694102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:24.843057259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:25:24.84651936Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:25:25.313304539Z caller=checkpoint.go:372 msg="writing series" size="25 MB"
level=debug ts=2026-05-10T20:25:25.313329321Z caller=checkpoint.go:384 msg="flushing series" totalSize="25 MB" series=7
level=debug ts=2026-05-10T20:25:26.445439427Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-05-10T20:25:27.21141596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:25:27.219702035Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-05-10T20:25:28.532169836Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-05-10T20:25:29.134273291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:29.262614327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:25:29.268134604Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-05-10T20:25:30.283565352Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-05-10T20:25:31.742555384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:25:31.747038934Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=debug ts=2026-05-10T20:25:31.786473453Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-05-10T20:25:32.083692205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:25:32.087880279Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:25:33.122182189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:25:33.127513453Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:25:33.261835464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:25:33.265314702Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-05-10T20:25:33.617421388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:25:33.62087457Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-05-10T20:25:33.856600896Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:33758"
level=warn ts=2026-05-10T20:25:34.133567634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:34.515384718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:25:34.520935507Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=debug ts=2026-05-10T20:25:35.039106399Z caller=checkpoint.go:372 msg="writing series" size="627 kB"
level=info ts=2026-05-10T20:25:36.284503127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:25:36.288592651Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=info ts=2026-05-10T20:25:37.629153719Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:25:37.632678304Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-05-10T20:25:38.291784324Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-05-10T20:25:39.134132592Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:25:41.544409661Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-05-10T20:25:41.915488582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:25:41.921228559Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-05-10T20:25:42.832759426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e2f32baf25a756c3 immediate=false num_chunks=1 total_comp="55 kB" avg_comp="55 kB" total_uncomp="191 kB" avg_uncomp="191 kB" full=1 labels="{stream=\"build-e2e-sspn\"}"
level=error ts=2026-05-10T20:25:42.836506669Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e2f32baf25a756c3/19e1343265d%3A19e135902ae%3A79c24ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-sspn\"}"
level=warn ts=2026-05-10T20:25:44.133682499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:25:44.797552425Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-05-10T20:25:47.72487239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:25:47.733220927Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:25:48.053566684Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=debug ts=2026-05-10T20:25:48.05359071Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-05-10T20:25:49.116965714Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:25:49.117372399Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=warn ts=2026-05-10T20:25:49.133694232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:49.133696644Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:25:49.133726745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:25:49.133738485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-05-10T20:25:49.434779121Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:25:49.982579686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:25:49.990884539Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-05-10T20:25:51.30278722Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:25:53.018031831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:25:53.023297157Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=warn ts=2026-05-10T20:25:54.133674055Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:25:54.506920582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:25:54.556286918Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=info ts=2026-05-10T20:25:57.377244261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:25:57.381039155Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-05-10T20:25:57.809309469Z caller=checkpoint.go:372 msg="writing series" size="6.2 kB"
level=info ts=2026-05-10T20:25:58.86217733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:25:58.86720737Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:25:58.867228794Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:25:58.867250925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:25:58.870920084Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-05-10T20:25:58.891997676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:25:58.898937955Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=warn ts=2026-05-10T20:25:59.134138662Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:25:59.573777235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:25:59.577183587Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-05-10T20:26:01.06246697Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-05-10T20:26:03.413909369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:26:03.417776555Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:26:03.417797417Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:26:03.417818552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:26:03.420580863Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-05-10T20:26:03.576064925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:26:03.580976516Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-05-10T20:26:03.860007608Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:48556"
level=info ts=2026-05-10T20:26:03.97169351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:26:03.975476179Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
level=warn ts=2026-05-10T20:26:04.13382339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:26:04.316040798Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-05-10T20:26:04.316062023Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-05-10T20:26:07.525563204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:26:07.53622362Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-05-10T20:26:07.56829932Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-05-10T20:26:08.844065389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:26:08.847580644Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-05-10T20:26:08.934832414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:26:08.938917723Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-05-10T20:26:09.134147264Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:10.3085608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:26:10.312394324Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:26:10.784477488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:26:10.788317624Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-05-10T20:26:10.825926518Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-05-10T20:26:10.825955977Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.5 MB" series=2
level=debug ts=2026-05-10T20:26:14.073975942Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-05-10T20:26:14.133780955Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:14.95870222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:26:14.962337979Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-05-10T20:26:15.96521418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:26:15.972772405Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-05-10T20:26:16.595923157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:26:16.600118247Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:26:17.328106841Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
ts=2026-05-10T20:26:18.073013703Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:53518"
level=info ts=2026-05-10T20:26:18.964284329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:26:18.967814159Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-05-10T20:26:19.134275896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:26:19.134303222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:26:19.134312352Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:26:19.134300351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:26:19.437628758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vlq94-e35ab537 10.130.0.76:7946"
level=debug ts=2026-05-10T20:26:20.036702493Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:26:20.580153719Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=info ts=2026-05-10T20:26:21.813325785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:26:21.817249479Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-05-10T20:26:22.082963526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:26:22.087062949Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-10T20:26:22.256839695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:26:22.260189546Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-05-10T20:26:23.833833522Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=warn ts=2026-05-10T20:26:24.133837876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:25.76768604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=7619766542bac838 immediate=false num_chunks=1 total_comp="406 kB" avg_comp="406 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"konflux-svlw\"}"
level=error ts=2026-05-10T20:26:25.77322086Z caller=flush.go:262 component=ingester loop=24 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/7619766542bac838/19e13444b55%3A19e134f69a6%3A5aab8ff1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw\"}"
level=info ts=2026-05-10T20:26:26.316306303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="37 kB" avg_uncomp="37 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-05-10T20:26:26.31993965Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19e1342e3f5%3A19e134408ee%3A61d96238?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-05-10T20:26:27.086449256Z caller=checkpoint.go:372 msg="writing series" size="174 kB"
level=info ts=2026-05-10T20:26:27.832406059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:26:27.836534905Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=info ts=2026-05-10T20:26:28.118408119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="551 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-05-10T20:26:28.126707147Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19e1332fa99%3A19e1342d719%3A4d4586d1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-05-10T20:26:28.561409386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-05-10T20:26:29.133765618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:30.129769471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=347d24d680fe556d immediate=false num_chunks=1 total_comp="597 kB" avg_comp="597 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" full=1 labels="{stream=\"build-e2e-nlva\"}"
level=error ts=2026-05-10T20:26:30.134225951Z caller=flush.go:262 component=ingester loop=13 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/347d24d680fe556d/19e13441c75%3A19e135500ad%3A6fb84c6e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nlva\"}"
level=debug ts=2026-05-10T20:26:30.340004544Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-05-10T20:26:31.321990096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.4 MB" avg_comp="2.2 MB" total_uncomp="30 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-05-10T20:26:31.329478354Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19e1332f52c%3A19e13502984%3A73763ee6?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-05-10T20:26:31.506877858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dca7b5caec2a9509 immediate=false num_chunks=1 total_comp="400 kB" avg_comp="400 kB" total_uncomp="1.8 MB" avg_uncomp="1.8 MB" full=1 labels="{stream=\"integration1-xjus\"}"
level=error ts=2026-05-10T20:26:31.511061374Z caller=flush.go:262 component=ingester loop=9 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dca7b5caec2a9509/19e13434445%3A19e134f13b6%3A5b3435da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-xjus\"}"
level=debug ts=2026-05-10T20:26:33.593222009Z caller=checkpoint.go:372 msg="writing series" size="292 kB"
ts=2026-05-10T20:26:33.862958299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.76:56884"
level=warn ts=2026-05-10T20:26:34.133677633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:36.748243134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=e2f32baf25a756c3 immediate=false num_chunks=1 total_comp="55 kB" avg_comp="55 kB" total_uncomp="191 kB" avg_uncomp="191 kB" full=1 labels="{stream=\"build-e2e-sspn\"}"
level=error ts=2026-05-10T20:26:36.752025997Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/e2f32baf25a756c3/19e1343265d%3A19e135902ae%3A79c24ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-sspn\"}"
level=debug ts=2026-05-10T20:26:36.845738288Z caller=checkpoint.go:372 msg="writing series" size="401 kB"
level=debug ts=2026-05-10T20:26:36.845920224Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=8
level=warn ts=2026-05-10T20:26:39.134004216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:39.70192177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=fc6ac7c1bc544cbe immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="14 kB" avg_uncomp="14 kB" full=1 labels="{stream=\"build-e2e-nbyi\"}"
level=error ts=2026-05-10T20:26:39.70642565Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/fc6ac7c1bc544cbe/19e1343ccfe%3A19e1345d5de%3A13e3440a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-nbyi\"}"
level=debug ts=2026-05-10T20:26:40.099826645Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=info ts=2026-05-10T20:26:40.262545759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="575 kB" avg_comp="575 kB" total_uncomp="3.6 MB" avg_uncomp="3.6 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-05-10T20:26:40.267117857Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19e13337513%3A19e137c9fde%3A4b70ce09?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-05-10T20:26:43.352117365Z caller=checkpoint.go:372 msg="writing series" size="429 kB"
level=debug ts=2026-05-10T20:26:43.352140772Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=2
level=warn ts=2026-05-10T20:26:44.133751512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:44.690682196Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d5e707f61a849bf3 immediate=false num_chunks=1 total_comp="13 kB" avg_comp="13 kB" total_uncomp="41 kB" avg_uncomp="41 kB" full=1 labels="{stream=\"konflux-svlw-managed\"}"
level=error ts=2026-05-10T20:26:44.694800331Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d5e707f61a849bf3/19e134be434%3A19e134cf8a7%3A4796047f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-svlw-managed\"}"
ts=2026-05-10T20:26:45.289745544Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.66:57632"
level=debug ts=2026-05-10T20:26:45.64957899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:26:46.6046587Z caller=checkpoint.go:372 msg="writing series" size="274 kB"
ts=2026-05-10T20:26:48.075281567Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.69:40088"
level=info ts=2026-05-10T20:26:49.117014914Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-05-10T20:26:49.117953182Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.79:11211 count=1
level=info ts=2026-05-10T20:26:49.134204741Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-05-10T20:26:49.134234976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-05-10T20:26:49.134243633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-05-10T20:26:49.134242218Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-05-10T20:26:49.440496414Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-f116cf12 10.128.0.69:7946"
level=info ts=2026-05-10T20:26:49.529179611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="496 kB" avg_comp="496 kB" total_uncomp="5.1 MB" avg_uncomp="5.1 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-05-10T20:26:49.533546292Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19e1332f3f0%3A19e13506ba2%3A96fd2684?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-05-10T20:26:49.857600825Z caller=checkpoint.go:372 msg="writing series" size="153 kB"
level=info ts=2026-05-10T20:26:51.104538042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.5 kB" avg_comp="6.5 kB" total_uncomp="32 kB" avg_uncomp="32 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-05-10T20:26:51.108686384Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19e1333028e%3A19e1333a709%3Ac275b6f1?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-05-10T20:26:51.419683235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="47 kB" avg_uncomp="47 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-05-10T20:26:51.423545433Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19e1332fa2c%3A19e133397b4%3A9c4edb5a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-05-10T20:26:52.440594982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ddf7ae5f420eea87 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-rvbr\"}"
level=error ts=2026-05-10T20:26:52.44847404Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ddf7ae5f420eea87/19e1342c89d%3A19e1350d106%3A15dd4379?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-rvbr\"}"
level=debug ts=2026-05-10T20:26:53.110552172Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-05-10T20:26:53.707196604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-05-10T20:26:53.710612011Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19e13337513%3A19e1333a709%3Af34f4272?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-05-10T20:26:54.133909639Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:54.418774583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=9 total_comp="24 MB" avg_comp="2.7 MB" total_uncomp="158 MB" avg_uncomp="18 MB" full=8 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-05-10T20:26:54.428923408Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19e13330ab2%3A19e1356d0fe%3Aa4ec743?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-05-10T20:26:55.00718748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-05-10T20:26:55.010617717Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19e1333a708%3A19e1333a709%3A4845c791?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-05-10T20:26:55.441987204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:26:55.446432776Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:26:55.44645168Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:26:55.44647587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:26:55.450467232Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-05-10T20:26:56.275168219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-05-10T20:26:56.280257961Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19e133397b4%3A19e1333a709%3A778381a9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-05-10T20:26:56.363743666Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=warn ts=2026-05-10T20:26:59.13399413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:26:59.365697789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="664 B" avg_comp="664 B" total_uncomp="1.9 kB" avg_uncomp="1.9 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-05-10T20:26:59.369330809Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19e13337513%3A19e133379c6%3A4cffe167?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-05-10T20:26:59.61704862Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-05-10T20:27:01.354802582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="15 MB" avg_comp="2.6 MB" total_uncomp="133 MB" avg_uncomp="22 MB" full=4 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-05-10T20:27:01.362951153Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19e1332f704%3A19e13339747%3Afc3d3dae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-05-10T20:27:02.87337952Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-05-10T20:27:02.873403932Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=6
level=info ts=2026-05-10T20:27:03.205403454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:27:03.209584527Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:27:03.209609675Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-05-10T20:27:03.209649323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="449 kB" avg_comp="449 kB" total_uncomp="2.7 MB" avg_uncomp="2.7 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-05-10T20:27:03.213659292Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19e1332f8bd%3A19e1382359e%3A5e63c2e5?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-05-10T20:27:03.835488318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-05-10T20:27:03.838884288Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19e133376b7%3A19e1333a985%3A6d2c2f79?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-05-10T20:27:03.917285661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.3 kB" avg_uncomp="3.3 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-05-10T20:27:03.920996888Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19e133376b7%3A19e134ddb36%3A22fbf975?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-05-10T20:27:04.13341928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-05-10T20:27:05.036697226Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="505 kB" avg_comp="505 kB" total_uncomp="4.8 MB" avg_uncomp="4.8 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-05-10T20:27:05.040739282Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19e1332f3d6%3A19e138a01b6%3A41db4c08?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-05-10T20:27:05.751995479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="816 kB" avg_comp="816 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-05-10T20:27:05.756561098Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19e1332f374%3A19e137b9795%3Aad6d47b8?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-05-10T20:27:06.123180487Z caller=checkpoint.go:372 msg="writing series" size="351 kB"
level=warn ts=2026-05-10T20:27:09.133681095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:27:09.375616329Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-05-10T20:27:11.621618655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6e62c9d9506cc83b immediate=false num_chunks=1 total_comp="396 kB" avg_comp="396 kB" total_uncomp="2.4 MB" avg_uncomp="2.4 MB" synced=1 labels="{stream=\"openshift-monitoring\"}"
level=error ts=2026-05-10T20:27:11.625471036Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6e62c9d9506cc83b/19e1332f344%3A19e1382453d%3Abed6e924?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-monitoring\"}"
level=debug ts=2026-05-10T20:27:11.917825118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-05-10T20:27:12.629339579Z caller=checkpoint.go:372 msg="writing series" size="559 kB"
level=debug ts=2026-05-10T20:27:12.629364481Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-05-10T20:27:12.634954285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="32 kB" avg_comp="32 kB" total_uncomp="165 kB" avg_uncomp="165 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-05-10T20:27:12.638161359Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19e13339777%3A19e13346cd5%3A85886ae3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-05-10T20:27:13.59990155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-05-10T20:27:13.601741549Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-05-10T20:27:14.133956669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-05-10T20:27:15.881921232Z caller=checkpoint.go:372 msg="writing series" size="259 kB"