level=info ts=2026-04-28T20:38:58.06913532Z caller=main.go:146 msg="Starting Loki" version="(version=3.7.1, branch=release-3.7.x, revision=2c8fff22)"
level=info ts=2026-04-28T20:38:58.069174239Z caller=main.go:147 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-28T20:38:58.070237638Z caller=modules.go:2203 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-28T20:38:58.071390507Z caller=server.go:397 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T20:38:58.071982527Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T20:38:58.072067854Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
level=info ts=2026-04-28T20:38:58.072565279Z caller=memberlist_client.go:517 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-b169d989
ts=2026-04-28T20:38:58.072666771Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T20:38:58.072678823Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=debug ts=2026-04-28T20:38:58.072698332Z caller=tcp_transport.go:448 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.88 advertisePort=7946
level=info ts=2026-04-28T20:38:58.072809903Z caller=memberlist_client.go:708 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-28T20:38:58.077797793Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T20:38:58.077821555Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-28T20:38:58.078845582Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-28T20:38:58.078917349Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T20:38:58.079429594Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-28T20:38:58.079455917Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-28T20:38:58.079453631Z
level=debug ts=2026-04-28T20:38:58.079469913Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-28T20:38:58.07953169Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-28T20:38:58.079585124Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-28T20:38:58.079596223Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-28T20:38:58.079594436Z
level=debug ts=2026-04-28T20:38:58.079605508Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-04-28T20:38:58.079814715Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T20:38:58.079839863Z caller=memberlist_client.go:714 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:38:58.079858084Z caller=memberlist_client.go:724 msg="memberlist fast-join failed to reach minimum required seed nodes" joined_nodes=0 required_nodes=1 elapsed_time=7.049825ms
level=error ts=2026-04-28T20:38:58.079869976Z caller=memberlist_client.go:599 msg="failed to fast-join the memberlist cluster at startup" err="fast-join failed to reach minimum required seed nodes: joined 0, required 1"
level=info ts=2026-04-28T20:38:58.079878319Z caller=memberlist_client.go:746 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-28T20:38:58.081855134Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T20:38:58.081929996Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.88 inf=eth0
ts=2026-04-28T20:38:58.082781751Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:38:58.082800374Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T20:38:58.095750701Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-28T20:38:58.095771795Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-28T20:38:58.095786292Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=debug ts=2026-04-28T20:38:58.095798201Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=debug ts=2026-04-28T20:38:58.09580493Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-28T20:38:58.095809994Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-28T20:38:58.095814481Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-28T20:38:58.095834599Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T20:38:58.095855872Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-28T20:38:58.095864486Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-28T20:38:58.095870892Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-28T20:38:58.096097688Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-28T20:38:58.096106529Z caller=module_service.go:82 msg=starting module=ring
level=debug ts=2026-04-28T20:38:58.096113321Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-28T20:38:58.09612667Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-28T20:38:58.096141392Z caller=ring.go:390 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-28T20:38:58.09617155Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-28T20:38:58.096178135Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=info ts=2026-04-28T20:38:58.096182973Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-28T20:38:58.096287912Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-28T20:38:58.096315211Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-28T20:38:58.096318453Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-28T20:38:58.096321219Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-28T20:38:58.096323788Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=info ts=2026-04-28T20:38:58.096326039Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-28T20:38:58.096349379Z caller=ingester.go:571 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-28T20:38:58.096359444Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T20:38:58.096407566Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=22.272µs success=true
level=info ts=2026-04-28T20:38:58.096420721Z caller=recovery.go:45 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-28T20:38:58.096503943Z caller=ingester.go:587 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=158.332µs errors=false
level=info ts=2026-04-28T20:38:58.096509804Z caller=ingester.go:593 component=ingester msg="recovering from WAL"
level=info ts=2026-04-28T20:38:58.096655999Z caller=ingester.go:609 component=ingester msg="WAL segment recovery finished" elapsed=309.095µs errors=false
level=info ts=2026-04-28T20:38:58.096669197Z caller=ingester.go:557 component=ingester msg="closing recoverer"
level=info ts=2026-04-28T20:38:58.096674863Z caller=ingester.go:565 component=ingester msg="WAL recovery finished" time=329.12µs
level=info ts=2026-04-28T20:38:58.096697335Z caller=wal.go:173 msg=started component=wal
level=info ts=2026-04-28T20:38:58.09699073Z caller=lifecycler.go:694 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T20:38:58.09699363Z caller=ingester.go:778 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.357142511s
level=info ts=2026-04-28T20:38:58.097012384Z caller=lifecycler.go:721 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-28T20:38:58.097075978Z caller=ingester.go:473 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-28T20:38:58.097083078Z caller=loki.go:615 msg="Loki started" startup_time=93.43741ms
level=debug ts=2026-04-28T20:38:58.097093646Z caller=lifecycler.go:558 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-28T20:38:58.097116572Z caller=lifecycler.go:562 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-28T20:38:58.097162641Z caller=memberlist_client.go:1252 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-28T20:38:59.366118079Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:38:59.366146729Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T20:39:02.016483707Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:39:02.016511595Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:39:03.097939447Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:39:07.703525864Z caller=memberlist_logger.go:74 level=warn msg="failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:39:07.703547673Z caller=memberlist_client.go:780 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:39:08.098075267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:39:12.163072219Z caller=logging.go:137 msg="GET /ready (503) 95.048µs"
level=warn ts=2026-04-28T20:39:13.09735777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:39:18.097554817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:39:20.919818559Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.66:7946: dial tcp 10.130.0.66:7946: connect: connection refused"
ts=2026-04-28T20:39:20.921115759Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.65:7946: dial tcp 10.130.0.65:7946: connect: connection refused"
ts=2026-04-28T20:39:20.922361172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-04-28T20:39:20.92475695Z caller=memberlist_logger.go:74 level=debug msg="failed to join 10.130.0.69:7946: dial tcp 10.130.0.69:7946: connect: connection refused"
ts=2026-04-28T20:39:20.925972841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.68:7946"
level=info ts=2026-04-28T20:39:20.927234428Z caller=memberlist_client.go:753 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=2 elapsed_time=22.847346789s
level=debug ts=2026-04-28T20:39:22.16200363Z caller=logging.go:137 msg="GET /ready (503) 71.009µs"
level=warn ts=2026-04-28T20:39:23.097944915Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:39:28.09743734Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-28T20:39:28.097463331Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-28T20:39:28.097472657Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:39:28.097507727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:39:33.098112464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:39:37.679034591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T20:39:37.679091775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:39:37.687106418Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:39:38.097524116Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:39:42.528759019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:39:43.09756216Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:39:43.965217Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37582"
ts=2026-04-28T20:39:45.469786499Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55696"
level=warn ts=2026-04-28T20:39:48.098069882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:39:51.961609102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:39:51.967818333Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:39:53.097145875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:39:56.506059014Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T20:39:58.079059296Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:39:58.079537354Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:39:58.098008475Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:39:58.098026929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:39:58.09803426Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:39:58.098051581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:40:03.097867552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:40:08.097260037Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:40:09.28990263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:40:12.530868035Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:40:13.098099231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:40:13.966786777Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33354"
ts=2026-04-28T20:40:15.472255425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35992"
level=warn ts=2026-04-28T20:40:18.097790181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:40:19.257583731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:40:23.097419603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:40:24.334308432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:40:24.34649097Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T20:40:28.097127749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:40:28.097162865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T20:40:28.09716618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:40:28.097172835Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:40:33.098014689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:40:36.403450335Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:40:38.097743119Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:40:42.532207875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:40:43.097697671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:40:44.463803346Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:40:48.09780958Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:40:53.097506225Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:40:56.547121934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-28T20:40:58.07898636Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:40:58.079006757Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:40:58.097092271Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:40:58.097111446Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:40:58.097119818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:40:58.097160687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:03.098124564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:08.09746329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:41:09.612862601Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:41:12.534638986Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:41:13.098196566Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:41:15.475592278Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56168"
level=info ts=2026-04-28T20:41:16.842600631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:41:16.84969003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:41:18.097143569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:23.098029058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:41:28.097527676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:41:28.097555315Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:41:28.097566993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:41:28.097593168Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:41:29.647505446Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T20:41:29.649231123Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:41:33.097650153Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:38.09792777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:41:39.791139811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:41:40.055069479Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51690"
ts=2026-04-28T20:41:42.536286072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:41:43.098051546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:48.09787739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:41:53.09734267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:41:58.079631706Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:41:58.080684072Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:41:58.09720634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T20:41:58.097210594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:41:58.09722887Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:41:58.097236208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:42:03.09778729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:42:08.098037979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:42:12.53854173Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:42:13.097970969Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:42:13.97491126Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45098"
level=debug ts=2026-04-28T20:42:14.470870178Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T20:42:15.193094209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:42:15.201920265Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T20:42:15.478770232Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47302"
level=debug ts=2026-04-28T20:42:16.510056217Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:42:18.097963334Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:42:23.098154027Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:42:28.097906055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:42:28.097940788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:42:28.097948897Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:42:28.097987312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:42:33.097687465Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:42:38.097131034Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:42:42.54013377Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:42:43.097309158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:42:43.976508156Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52522"
ts=2026-04-28T20:42:45.481298927Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56262"
level=debug ts=2026-04-28T20:42:46.170719223Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:42:48.097134072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:42:53.098166738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:42:58.07921365Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:42:58.07978119Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:42:58.097221438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T20:42:58.097224952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:42:58.097238859Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:42:58.097247671Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:43:03.097676549Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:43:04.613649185Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:43:04.620690504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:43:08.098066852Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:43:12.541484971Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=debug ts=2026-04-28T20:43:12.966507079Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:43:13.098044875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:43:17.011049322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:43:18.09718607Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:43:23.097311768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:43:28.097402127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T20:43:28.097419435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:43:28.097433002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:43:28.097450086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:43:33.097403591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:43:38.098044394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:43:42.543420181Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:43:43.09769152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:43:43.979990326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49888"
ts=2026-04-28T20:43:45.485069451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51462"
level=warn ts=2026-04-28T20:43:48.098004936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:43:48.258349628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:43:48.268293319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:43:50.039267842Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:43:53.097811706Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:43:58.078970403Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:43:58.079567596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:43:58.096761517Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T20:43:58.096958643Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-28T20:43:58.096967606Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T20:43:58.096982912Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=10.8µs success=true
level=debug ts=2026-04-28T20:43:58.097146816Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-28T20:43:58.097165134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:43:58.097175854Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:43:58.097191497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:43:58.097200267Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:44:02.452900482Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T20:44:03.097905203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:06.807388592Z caller=checkpoint.go:372 msg="writing series" size="416 kB"
level=warn ts=2026-04-28T20:44:08.097514604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:11.162153465Z caller=checkpoint.go:372 msg="writing series" size="338 kB"
ts=2026-04-28T20:44:12.545467682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:44:13.097672741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:14.989870497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:44:15.517643407Z caller=checkpoint.go:372 msg="writing series" size="8.5 kB"
level=warn ts=2026-04-28T20:44:18.098095174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:19.660707268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T20:44:19.872396585Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-04-28T20:44:23.097953963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:24.227185372Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T20:44:28.097927725Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T20:44:28.097935471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:28.097951008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:44:28.097958495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:44:28.581923245Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-28T20:44:28.58194162Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=8
level=debug ts=2026-04-28T20:44:32.936014001Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=warn ts=2026-04-28T20:44:33.097725082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:37.291704325Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-04-28T20:44:38.097484211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:44:39.668933103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:44:39.678275039Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T20:44:40.066033907Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51418"
level=debug ts=2026-04-28T20:44:41.645755695Z caller=checkpoint.go:372 msg="writing series" size="47 kB"
ts=2026-04-28T20:44:42.5477336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:44:43.097580839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:44.983769435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:44:46.000884622Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-04-28T20:44:48.097144432Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:50.356238901Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-04-28T20:44:53.097132884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:44:54.710731991Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=info ts=2026-04-28T20:44:58.079643385Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:44:58.080951682Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T20:44:58.097193674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:44:58.097199981Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:44:58.097217185Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:44:58.097224735Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:44:59.066040581Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=warn ts=2026-04-28T20:45:03.098128099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:03.420417733Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-04-28T20:45:07.774645332Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-28T20:45:08.09807636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:09.688033263Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:45:10.068369128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60378"
level=debug ts=2026-04-28T20:45:12.130126533Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
ts=2026-04-28T20:45:12.550035364Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:45:13.097348604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:16.485034547Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=warn ts=2026-04-28T20:45:18.097208519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:45:20.34219329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:45:20.349231912Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:45:20.839609954Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T20:45:20.915745889Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T20:45:23.098108949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:25.194843389Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T20:45:28.097319499Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:45:28.097342326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:45:28.097349881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:45:28.097347788Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:29.549029195Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T20:45:33.097163251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:33.90365055Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T20:45:34.96167517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:45:38.097261677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:38.258859271Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=debug ts=2026-04-28T20:45:38.258889092Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=16
ts=2026-04-28T20:45:42.551581534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=debug ts=2026-04-28T20:45:42.61375768Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T20:45:43.097345042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:46.968172044Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-28T20:45:48.097892782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:51.323035912Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T20:45:53.097683564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:45:55.678092902Z caller=checkpoint.go:372 msg="writing series" size="229 kB"
level=info ts=2026-04-28T20:45:58.078976363Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:45:58.079392949Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-04-28T20:45:58.08068849Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T20:45:58.080726333Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974898 n=1
level=debug ts=2026-04-28T20:45:58.080765742Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974898 n=1
level=info ts=2026-04-28T20:45:58.097988328Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:45:58.098013167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:45:58.098022106Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:45:58.098023942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:00.032925459Z caller=checkpoint.go:372 msg="writing series" size="524 kB"
level=debug ts=2026-04-28T20:46:00.716601507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:46:03.097941377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:04.388148162Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T20:46:08.097498717Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:08.742462908Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
ts=2026-04-28T20:46:12.553509651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:46:13.09782358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:13.097908862Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-04-28T20:46:13.097922229Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-04-28T20:46:13.989546355Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52032"
level=debug ts=2026-04-28T20:46:14.496049486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T20:46:14.497604977Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T20:46:17.452916483Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-04-28T20:46:18.085933222Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:46:18.094691443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:46:18.094714094Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T20:46:18.094738271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:46:18.097950768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=error ts=2026-04-28T20:46:18.101404372Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:46:21.807753949Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-04-28T20:46:23.09813696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:26.161878016Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=warn ts=2026-04-28T20:46:28.097441302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:46:28.097444312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:46:28.097464629Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:46:28.09747314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T20:46:28.329030883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="26 MB" avg_uncomp="26 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:46:28.338744763Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:46:30.516804754Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T20:46:33.097724385Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:34.349253579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:46:34.871737205Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=warn ts=2026-04-28T20:46:38.097879528Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:39.226579056Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-28T20:46:40.073009824Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44892"
ts=2026-04-28T20:46:42.554867027Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:46:43.098075268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:43.581435095Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-28T20:46:47.935954132Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=warn ts=2026-04-28T20:46:48.097483312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:46:51.539981357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:46:51.553747379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:46:52.291586272Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-04-28T20:46:53.097757017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:46:56.646223893Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T20:46:58.078984987Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:46:58.079249066Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:46:58.097199905Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:46:58.097216551Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:46:58.09722368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:46:58.097300543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:01.000421023Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-04-28T20:47:03.097969818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:03.972037074Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T20:47:05.355790994Z caller=checkpoint.go:372 msg="writing series" size="185 kB"
level=debug ts=2026-04-28T20:47:06.756718239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:47:08.097165777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:09.710684294Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
ts=2026-04-28T20:47:12.556309747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:47:13.097807884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:47:13.993083158Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41056"
level=debug ts=2026-04-28T20:47:14.065476212Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
ts=2026-04-28T20:47:15.497752207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50706"
level=warn ts=2026-04-28T20:47:18.097705387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:18.420831955Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=debug ts=2026-04-28T20:47:22.774802993Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=debug ts=2026-04-28T20:47:22.774825775Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-04-28T20:47:23.09778935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:27.129995755Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-04-28T20:47:28.097880285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:47:28.097904728Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:47:28.097913472Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:47:28.097907399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:31.484294134Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T20:47:33.097872917Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:34.245396806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:47:35.839984901Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-04-28T20:47:38.097991963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:47:40.076868103Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59940"
level=debug ts=2026-04-28T20:47:40.19454731Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-04-28T20:47:42.558112825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:47:43.098112018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:44.549165362Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-04-28T20:47:48.097387945Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:48.904041558Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-04-28T20:47:51.553495402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:47:51.576242032Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:47:53.098187507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:47:53.258796305Z caller=checkpoint.go:372 msg="writing series" size="289 kB"
level=debug ts=2026-04-28T20:47:57.234040222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T20:47:57.618716159Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-04-28T20:47:57.618737804Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.8 MB" series=8
level=info ts=2026-04-28T20:47:58.079255702Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:47:58.079671383Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T20:47:58.09716347Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:47:58.097172786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:47:58.097194537Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:47:58.097202574Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:48:01.969008968Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=warn ts=2026-04-28T20:48:03.097477809Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:06.323227943Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T20:48:08.097881499Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:09.613875234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:48:10.678194731Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-04-28T20:48:12.560370265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:48:13.097597593Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:48:13.995462563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37176"
level=debug ts=2026-04-28T20:48:15.032886222Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-04-28T20:48:15.501606619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48922"
level=warn ts=2026-04-28T20:48:18.097928155Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:19.388164491Z caller=checkpoint.go:372 msg="writing series" size="9.8 kB"
level=warn ts=2026-04-28T20:48:23.097929443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:23.742768573Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T20:48:28.097900394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:48:28.097916546Z caller=checkpoint.go:384 msg="flushing series" totalSize="221 kB" series=6
level=debug ts=2026-04-28T20:48:28.097930121Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:48:28.097939025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:48:28.097987919Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:48:28.10790242Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-28T20:48:28.111668643Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.014565852s
level=info ts=2026-04-28T20:48:31.9858395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:48:31.996323407Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:48:33.097750229Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:48:38.097958133Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:48:42.562081876Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:48:43.097162213Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:43.380893612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:48:48.097862512Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:48:53.097754899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:48:56.42929174Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T20:48:58.078949155Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:48:58.079274619Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:48:58.096855546Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T20:48:58.097009208Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-28T20:48:58.097017686Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T20:48:58.097035322Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.043µs success=true
level=info ts=2026-04-28T20:48:58.097095769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:48:58.097115985Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:48:58.097124435Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:48:58.097193828Z caller=checkpoint.go:372 msg="writing series" size="1.6 kB"
level=warn ts=2026-04-28T20:48:58.09720995Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:02.452423755Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T20:49:03.097478384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:06.80757134Z caller=checkpoint.go:372 msg="writing series" size="428 kB"
level=warn ts=2026-04-28T20:49:08.097645875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:11.16283802Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-04-28T20:49:12.413360912Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:49:12.421959581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T20:49:12.563524433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:49:13.097896363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:13.382500819Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:49:13.998725226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54072"
level=debug ts=2026-04-28T20:49:15.516809416Z caller=checkpoint.go:372 msg="writing series" size="8.5 kB"
level=warn ts=2026-04-28T20:49:18.097374927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:19.872233737Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=warn ts=2026-04-28T20:49:23.098120819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:24.227117405Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T20:49:28.097446481Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:49:28.097473795Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:49:28.097482168Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:49:28.097522851Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:28.582025936Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-28T20:49:28.582049999Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=8
level=debug ts=2026-04-28T20:49:32.937107603Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=warn ts=2026-04-28T20:49:33.097582527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:37.290916637Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=debug ts=2026-04-28T20:49:37.504659184Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:49:38.09756101Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:41.646609485Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
ts=2026-04-28T20:49:42.565453555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:49:43.101335352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:46.001376537Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=warn ts=2026-04-28T20:49:48.097400574Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:50.356083298Z caller=checkpoint.go:372 msg="writing series" size="18 kB"
level=debug ts=2026-04-28T20:49:51.359087069Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:49:53.098001114Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:54.711215915Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-04-28T20:49:58.07886843Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:49:58.079914464Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T20:49:58.097634397Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:49:58.097659065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:49:58.09766682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:49:58.097668642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:49:59.065799755Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=info ts=2026-04-28T20:50:00.146888454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:50:00.155070287Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:50:03.098167702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:03.420655423Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=debug ts=2026-04-28T20:50:07.205149035Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:50:07.774931459Z caller=checkpoint.go:372 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-28T20:50:08.098129968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:12.130421793Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
ts=2026-04-28T20:50:12.56744732Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:50:13.097772523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:50:14.001952941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52062"
level=debug ts=2026-04-28T20:50:16.484976579Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=debug ts=2026-04-28T20:50:16.485000213Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-28T20:50:18.09774609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:20.839891356Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-04-28T20:50:23.097923537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:25.194997245Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T20:50:28.097205149Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:50:28.09723025Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:50:28.097238153Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:50:28.097273379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:29.549748855Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T20:50:33.097765284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:33.903701536Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T20:50:36.192607296Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:50:36.194327705Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T20:50:38.097115966Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:38.259661961Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=debug ts=2026-04-28T20:50:39.867721759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:50:40.086211518Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48002"
ts=2026-04-28T20:50:42.569752685Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=debug ts=2026-04-28T20:50:42.613811304Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T20:50:43.097982263Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:46.968231948Z caller=checkpoint.go:372 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-28T20:50:48.097146618Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:51.323839006Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T20:50:53.097845136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:55.678012966Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-04-28T20:50:58.078957784Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:50:58.081099801Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:50:58.098050939Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T20:50:58.098054345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:50:58.098070443Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:50:58.098078161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T20:50:58.99982879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:50:59.010873905Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:51:00.033632549Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=debug ts=2026-04-28T20:51:00.033654301Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
level=warn ts=2026-04-28T20:51:03.098231577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:04.388553347Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T20:51:08.097397166Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:08.743204894Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=debug ts=2026-04-28T20:51:09.631720347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:51:10.088440877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40934"
ts=2026-04-28T20:51:12.571795051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:51:13.097641979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:13.097680058Z caller=checkpoint.go:372 msg="writing series" size="164 kB"
level=debug ts=2026-04-28T20:51:17.452268333Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-04-28T20:51:18.097710237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:21.609116806Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T20:51:21.807851691Z caller=checkpoint.go:372 msg="writing series" size="302 kB"
level=warn ts=2026-04-28T20:51:23.097861358Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:26.161841777Z caller=checkpoint.go:372 msg="writing series" size="171 kB"
level=info ts=2026-04-28T20:51:28.097765775Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:51:28.097792008Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:51:28.097802297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:51:28.097808439Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:30.517390472Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T20:51:33.097421589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:34.872041821Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-28T20:51:36.640471142Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:51:38.097371007Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:39.227295807Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-28T20:51:40.090643965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43982"
level=info ts=2026-04-28T20:51:42.556013028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:51:42.568271501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T20:51:42.574002949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:51:43.098014911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:43.581249737Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=debug ts=2026-04-28T20:51:47.936976312Z caller=checkpoint.go:372 msg="writing series" size="78 kB"
level=warn ts=2026-04-28T20:51:48.097954181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:52.291403683Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-04-28T20:51:53.097576478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:51:56.646375133Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-28T20:51:58.079470701Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:51:58.080651416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:51:58.097851592Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:51:58.097874964Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:51:58.09788238Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:51:58.097900254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:01.000437397Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
level=debug ts=2026-04-28T20:52:01.00045385Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-04-28T20:52:03.097867844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:05.356091466Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=debug ts=2026-04-28T20:52:06.311367866Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:52:08.097174399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:09.710803284Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T20:52:11.794842519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:52:12.576244005Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:52:13.098165386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:14.065267275Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-28T20:52:18.097214887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:18.420601689Z caller=checkpoint.go:372 msg="writing series" size="191 kB"
level=debug ts=2026-04-28T20:52:22.774886928Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-04-28T20:52:23.097966486Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:27.130414078Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-04-28T20:52:28.097142573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:52:28.097167527Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:52:28.097174939Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:52:28.09717031Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:31.484438147Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T20:52:33.097394025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:52:33.423408691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:52:33.432472128Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:52:33.432498578Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T20:52:33.432523578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:52:33.439406794Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:52:35.806129041Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:52:35.839196982Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-04-28T20:52:38.097604469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:40.19438046Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-04-28T20:52:42.57839041Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:52:43.097887093Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:44.549879946Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-04-28T20:52:45.508753706Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T20:52:45.518724549Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53534"
level=warn ts=2026-04-28T20:52:48.097205245Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:52:48.437409004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:52:48.447316111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:52:48.904015073Z caller=checkpoint.go:372 msg="writing series" size="29 kB"
level=warn ts=2026-04-28T20:52:53.097716506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:52:53.25925139Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=debug ts=2026-04-28T20:52:53.259273758Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=debug ts=2026-04-28T20:52:57.618960321Z caller=checkpoint.go:372 msg="writing series" size="6.9 MB"
level=debug ts=2026-04-28T20:52:57.618983346Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.9 MB" series=1
level=info ts=2026-04-28T20:52:58.079297417Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:52:58.079693011Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:52:58.097348128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:52:58.097372474Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:52:58.09738014Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:52:58.097421384Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:53:01.969200978Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=warn ts=2026-04-28T20:53:03.097591696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:53:04.059240474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:53:06.323582636Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T20:53:08.098050067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:53:10.678802053Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
ts=2026-04-28T20:53:12.580908103Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:53:13.098165522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:53:15.033186139Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-28T20:53:18.097183909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:53:18.469280381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:53:18.480739186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:53:19.387641398Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T20:53:23.097654929Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:53:23.743213451Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-28T20:53:28.097337578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:53:28.097344807Z caller=checkpoint.go:384 msg="flushing series" totalSize="258 kB" series=6
level=debug ts=2026-04-28T20:53:28.097359749Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:53:28.097367204Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:53:28.097393136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:53:28.10209439Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-28T20:53:28.103559178Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006404158s
level=debug ts=2026-04-28T20:53:32.09997048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:53:33.097713016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:53:38.097753069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:53:42.583492452Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:53:43.097453979Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:53:44.016932367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57060"
ts=2026-04-28T20:53:45.522228451Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53564"
level=debug ts=2026-04-28T20:53:48.000271903Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:53:48.097653721Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:53:53.097447131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:53:58.079106602Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:53:58.079568945Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:53:58.097113908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:53:58.097132787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:53:58.097128934Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T20:53:58.097140368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:53:58.097158572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:53:58.097264079Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-28T20:53:58.097272915Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T20:53:58.097289191Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.995µs success=true
level=debug ts=2026-04-28T20:53:58.097443682Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-04-28T20:54:01.691393534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:54:01.705838422Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:54:02.452956125Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T20:54:03.097752173Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:03.92322579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:54:06.80805106Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=warn ts=2026-04-28T20:54:08.097140562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:11.162342596Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
ts=2026-04-28T20:54:12.586120519Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:54:13.097959391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:54:14.018284681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:53916"
level=debug ts=2026-04-28T20:54:15.516848141Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T20:54:18.097517638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:19.872353579Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=warn ts=2026-04-28T20:54:23.097364238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:24.226511693Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T20:54:28.097788896Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:54:28.097813702Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:54:28.09782157Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:54:28.097815714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:28.3874748Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:54:28.583390654Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-28T20:54:28.583411847Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=8
level=debug ts=2026-04-28T20:54:32.937177863Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=warn ts=2026-04-28T20:54:33.097344814Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:37.291188646Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-04-28T20:54:38.09719968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:39.448943428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:54:39.45058811Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T20:54:41.646741147Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
ts=2026-04-28T20:54:42.58786453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:54:43.098121905Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:54:44.019546762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45980"
level=info ts=2026-04-28T20:54:45.950331632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:54:45.960950499Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:54:46.001219432Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=warn ts=2026-04-28T20:54:48.097161584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:50.356293767Z caller=checkpoint.go:372 msg="writing series" size="28 kB"
level=warn ts=2026-04-28T20:54:53.097725793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:54.711059824Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=info ts=2026-04-28T20:54:58.079977022Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:54:58.080829624Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:54:58.098063994Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:54:58.098080495Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:54:58.098088394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:54:58.098087389Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:54:59.066358757Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=debug ts=2026-04-28T20:54:59.066379882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-28T20:55:03.09820466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:03.420502035Z caller=checkpoint.go:372 msg="writing series" size="36 kB"
level=debug ts=2026-04-28T20:55:04.101158951Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:55:07.775862252Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-28T20:55:08.098005291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:12.129791704Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
ts=2026-04-28T20:55:12.5895552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:55:13.097376821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:16.485025369Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-04-28T20:55:18.098031974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:20.839764052Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=warn ts=2026-04-28T20:55:23.097586053Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:25.194585256Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T20:55:28.097812343Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:55:28.09784262Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:55:28.097853074Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:55:28.097847769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:29.549251199Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-28T20:55:31.464750564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:55:31.475952143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:55:33.09796113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:33.904267772Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T20:55:34.323716441Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:55:35.071870619Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:55:38.097462219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:38.259977359Z caller=checkpoint.go:372 msg="writing series" size="359 kB"
ts=2026-04-28T20:55:42.592057316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=debug ts=2026-04-28T20:55:42.614096398Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T20:55:43.097518526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:55:44.023006475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59956"
level=debug ts=2026-04-28T20:55:46.968984132Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T20:55:48.097190444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:51.324022882Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T20:55:53.098010973Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:55:55.678761659Z caller=checkpoint.go:372 msg="writing series" size="337 kB"
level=debug ts=2026-04-28T20:55:55.678787382Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-04-28T20:55:58.079226831Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:55:58.079785283Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:55:58.097658257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:55:58.097680284Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T20:55:58.097670977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:55:58.09769024Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T20:56:00.033951687Z caller=checkpoint.go:372 msg="writing series" size="513 kB"
level=debug ts=2026-04-28T20:56:00.662970158Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T20:56:00.663031677Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:56:00.674949215Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T20:56:03.097690008Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:04.388495278Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T20:56:08.098018868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:08.743224513Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=info ts=2026-04-28T20:56:11.440605358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.8 MB" avg_comp="2.8 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:56:11.451783666Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T20:56:12.593280419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:56:13.097702805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:13.097860786Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
ts=2026-04-28T20:56:15.530251452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39424"
level=debug ts=2026-04-28T20:56:17.452858469Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-04-28T20:56:18.097569702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:21.80751701Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-28T20:56:21.807539246Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-28T20:56:23.097718095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:56:24.472038073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:56:24.48366233Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:56:26.163140883Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=info ts=2026-04-28T20:56:28.097315675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:56:28.097350261Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:56:28.097363015Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:56:28.097383006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:30.517843334Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T20:56:33.097625754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:33.405570347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T20:56:34.872116543Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-28T20:56:35.390862887Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:56:38.097453495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:39.226749827Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
ts=2026-04-28T20:56:40.104757185Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34772"
ts=2026-04-28T20:56:42.595152772Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:56:43.097581625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:43.58158724Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T20:56:47.845263621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.7 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:56:47.854804549Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T20:56:47.937231536Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=warn ts=2026-04-28T20:56:48.097399426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:52.291790822Z caller=checkpoint.go:372 msg="writing series" size="58 kB"
level=warn ts=2026-04-28T20:56:53.097591584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:56:56.646798586Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-04-28T20:56:58.078964397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:56:58.079479947Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:56:58.098095003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:56:58.098115787Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:56:58.098124272Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:56:58.098146393Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:01.001218144Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=warn ts=2026-04-28T20:57:03.098001733Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:05.356118292Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T20:57:08.097751414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:09.710997944Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T20:57:10.981893037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:57:12.597323995Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:57:13.097162832Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:14.06593601Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=debug ts=2026-04-28T20:57:15.91360975Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:57:18.097979277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:18.420498121Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=info ts=2026-04-28T20:57:21.679132951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:57:21.686990649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T20:57:22.775321431Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=debug ts=2026-04-28T20:57:22.775342993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=14
level=warn ts=2026-04-28T20:57:23.09728268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:27.130359604Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=info ts=2026-04-28T20:57:28.097833151Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:57:28.097874137Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:57:28.097886257Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:57:28.097934665Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:31.4851722Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=info ts=2026-04-28T20:57:32.756536802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:57:32.770240108Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T20:57:33.098237012Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:35.839918018Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-04-28T20:57:38.097538982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:40.194410644Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
ts=2026-04-28T20:57:42.598945999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T20:57:43.097650145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:43.616533711Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:57:44.549967342Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=warn ts=2026-04-28T20:57:48.097842098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:48.904279831Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=warn ts=2026-04-28T20:57:53.098055313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:57:53.259684002Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=debug ts=2026-04-28T20:57:57.618463246Z caller=checkpoint.go:372 msg="writing series" size="7.3 MB"
level=debug ts=2026-04-28T20:57:57.618490923Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=8
level=info ts=2026-04-28T20:57:58.079134025Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:57:58.082331611Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:57:58.097438955Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:57:58.097471813Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:57:58.097482332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:57:58.097490158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:01.969430197Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-28T20:58:02.262593357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T20:58:03.097534723Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:06.323764684Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T20:58:08.097754897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:08.297776787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T20:58:10.109836922Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56502"
level=debug ts=2026-04-28T20:58:10.678245043Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
ts=2026-04-28T20:58:12.600673443Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T20:58:12.873407148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:58:12.883765409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:58:13.097477458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:15.033091765Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
ts=2026-04-28T20:58:15.539657404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:50700"
level=info ts=2026-04-28T20:58:16.887010368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.6 MB" avg_comp="2.9 MB" total_uncomp="72 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:58:16.894867509Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T20:58:18.097657021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:19.388286608Z caller=checkpoint.go:372 msg="writing series" size="79 kB"
level=warn ts=2026-04-28T20:58:23.097548139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:23.743375889Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=info ts=2026-04-28T20:58:28.097185576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:58:28.097219764Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:58:28.097230389Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:58:28.097245065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:28.098288398Z caller=checkpoint.go:384 msg="flushing series" totalSize="542 kB" series=6
level=info ts=2026-04-28T20:58:28.243354969Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-28T20:58:28.244969376Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.147567146s
level=warn ts=2026-04-28T20:58:33.097324476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:35.735355413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T20:58:38.097694984Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:58:40.112355519Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35718"
level=debug ts=2026-04-28T20:58:41.474971638Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T20:58:41.47681946Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T20:58:42.602719682Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:58:43.097143875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:58:44.036900408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55552"
level=warn ts=2026-04-28T20:58:48.097581052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T20:58:53.097882514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:58:53.286224463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:58:53.296579818Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:58:53.296610835Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T20:58:53.296658656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:58:53.305127285Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T20:58:58.07901487Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:58:58.079644255Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:58:58.097018565Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T20:58:58.097143094Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-28T20:58:58.097153669Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T20:58:58.097176595Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=16.936µs success=true
level=info ts=2026-04-28T20:58:58.097227521Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:58:58.097250356Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:58:58.097262063Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:58:58.09725689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:58:58.097361683Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T20:59:01.51777139Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T20:59:03.097686507Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T20:59:03.627680872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:59:03.638075955Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T20:59:04.946447777Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-04-28T20:59:04.946476695Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=3
level=debug ts=2026-04-28T20:59:06.502378275Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T20:59:07.444129696Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:59:07.451785344Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:59:08.097133182Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:08.351208601Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
ts=2026-04-28T20:59:10.11433064Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53132"
level=debug ts=2026-04-28T20:59:11.768314033Z caller=checkpoint.go:372 msg="writing series" size="8.6 kB"
ts=2026-04-28T20:59:12.605066038Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T20:59:13.097222944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T20:59:14.03829124Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56594"
level=debug ts=2026-04-28T20:59:15.186999296Z caller=checkpoint.go:372 msg="writing series" size="294 kB"
ts=2026-04-28T20:59:15.54318815Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42612"
level=warn ts=2026-04-28T20:59:18.097846641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:18.603743467Z caller=checkpoint.go:372 msg="writing series" size="106 kB"
level=debug ts=2026-04-28T20:59:22.021897153Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-04-28T20:59:23.097131444Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:25.440128567Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-28T20:59:25.440157362Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=6
level=info ts=2026-04-28T20:59:28.097098245Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:59:28.097126633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:59:28.097135235Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:59:28.097197791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:28.435604109Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T20:59:28.857219859Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=debug ts=2026-04-28T20:59:32.275711711Z caller=checkpoint.go:372 msg="writing series" size="496 kB"
level=warn ts=2026-04-28T20:59:33.098027869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:35.693279133Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-04-28T20:59:36.461510561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T20:59:36.474423058Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T20:59:38.097953314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:38.435779367Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T20:59:39.110469486Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
ts=2026-04-28T20:59:40.116594537Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34414"
level=debug ts=2026-04-28T20:59:42.528711938Z caller=checkpoint.go:372 msg="writing series" size="163 kB"
ts=2026-04-28T20:59:42.60756009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T20:59:43.097672719Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:45.945546384Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=info ts=2026-04-28T20:59:47.943690155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T20:59:47.955647596Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T20:59:48.097984343Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:49.363860194Z caller=checkpoint.go:372 msg="writing series" size="372 kB"
level=debug ts=2026-04-28T20:59:49.363886831Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-04-28T20:59:52.781216498Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=warn ts=2026-04-28T20:59:53.098165202Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:56.199401197Z caller=checkpoint.go:372 msg="writing series" size="395 kB"
level=info ts=2026-04-28T20:59:58.079382268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T20:59:58.079925914Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T20:59:58.097090442Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T20:59:58.097114076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T20:59:58.097121345Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T20:59:58.097180811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T20:59:59.617493971Z caller=checkpoint.go:372 msg="writing series" size="426 kB"
level=debug ts=2026-04-28T21:00:03.034306435Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:00:03.097943142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:06.452290262Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T21:00:08.097191238Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:09.870449985Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
ts=2026-04-28T21:00:10.119140653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45544"
level=debug ts=2026-04-28T21:00:11.864680368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:00:12.609870677Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:00:13.097950669Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:13.287563745Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
ts=2026-04-28T21:00:14.042178697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45332"
ts=2026-04-28T21:00:15.548655871Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40630"
level=debug ts=2026-04-28T21:00:16.705451427Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=warn ts=2026-04-28T21:00:18.097854563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:00:18.371609886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:00:18.380871771Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:00:19.981481648Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:00:20.122984719Z caller=checkpoint.go:372 msg="writing series" size="85 kB"
level=debug ts=2026-04-28T21:00:20.123007954Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-28T21:00:23.097131305Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:23.540797251Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-04-28T21:00:26.958798307Z caller=checkpoint.go:372 msg="writing series" size="7.8 kB"
level=info ts=2026-04-28T21:00:28.097979435Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:00:28.098005501Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:00:28.098013417Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:00:28.09801585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:30.376443004Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T21:00:33.097340481Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:33.794649223Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-28T21:00:36.930043967Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:00:37.212330321Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=warn ts=2026-04-28T21:00:38.097405866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:40.629244211Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
ts=2026-04-28T21:00:42.611356261Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:00:43.097444621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:44.047481971Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T21:00:47.465013244Z caller=checkpoint.go:372 msg="writing series" size="249 kB"
level=info ts=2026-04-28T21:00:47.540356453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:00:47.553763715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:00:48.097994363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:50.883131961Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T21:00:53.097929791Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:00:54.300690184Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-04-28T21:00:57.718626806Z caller=checkpoint.go:372 msg="writing series" size="27 kB"
level=info ts=2026-04-28T21:00:58.079852559Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:00:58.080806259Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T21:00:58.080848807Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974899 n=1
level=debug ts=2026-04-28T21:00:58.0808935Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974899 n=1
level=debug ts=2026-04-28T21:00:58.081131823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:00:58.097443788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:00:58.097466123Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:00:58.09747355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:00:58.097469683Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:01.136270241Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:01:03.097563068Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:04.553538521Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=debug ts=2026-04-28T21:01:07.971838169Z caller=checkpoint.go:372 msg="writing series" size="474 kB"
level=debug ts=2026-04-28T21:01:07.971862672Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=14
level=warn ts=2026-04-28T21:01:08.097288366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:01:10.123564602Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40566"
level=debug ts=2026-04-28T21:01:11.389047491Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=debug ts=2026-04-28T21:01:11.991903495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:01:12.613319129Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:01:13.097901086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:01:14.045699005Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:39646"
level=debug ts=2026-04-28T21:01:14.363208712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:01:14.672074596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:01:14.682743736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:01:14.807059618Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=warn ts=2026-04-28T21:01:18.097800148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:18.224130756Z caller=checkpoint.go:372 msg="writing series" size="26 kB"
level=debug ts=2026-04-28T21:01:21.642505071Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-04-28T21:01:23.097152938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:25.060445802Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-04-28T21:01:28.098037485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:01:28.098063816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:01:28.098073559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:01:28.098066679Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:28.477703632Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=debug ts=2026-04-28T21:01:31.895415398Z caller=checkpoint.go:372 msg="writing series" size="355 kB"
level=warn ts=2026-04-28T21:01:33.097838492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:35.31278154Z caller=checkpoint.go:372 msg="writing series" size="86 kB"
level=warn ts=2026-04-28T21:01:38.097802953Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:38.730478251Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-04-28T21:01:39.372757009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:01:39.387478202Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:01:42.14892288Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-28T21:01:42.158384793Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:01:42.615264785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:01:43.097837382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:45.566646353Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T21:01:45.566670707Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-28T21:01:48.097705924Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:48.983528594Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T21:01:52.401233001Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:01:53.098008406Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:55.81995193Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=info ts=2026-04-28T21:01:58.079268646Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:01:58.080194423Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:01:58.097671835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:01:58.097698086Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:01:58.097714704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:01:58.097724862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:01:59.237483128Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=debug ts=2026-04-28T21:02:02.654702524Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:02:03.09754363Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:06.072656663Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:02:08.097679029Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:09.490800635Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
ts=2026-04-28T21:02:10.126791157Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40426"
ts=2026-04-28T21:02:12.617704721Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:02:12.907729495Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T21:02:13.012539088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:02:13.021038532Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:02:13.09801379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:02:14.049805112Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54308"
level=debug ts=2026-04-28T21:02:14.949979013Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:02:15.306280314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:02:16.325831979Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T21:02:18.097216329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:19.743261671Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:02:23.097311865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:23.161506588Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=debug ts=2026-04-28T21:02:26.579288914Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=info ts=2026-04-28T21:02:28.097595158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:02:28.097596758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:28.097632678Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:02:28.09764064Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:02:29.996875248Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=info ts=2026-04-28T21:02:32.803861374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:02:32.813201625Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:02:32.813223987Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:02:32.813251567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:02:32.821501483Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:02:33.097928398Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:33.414177497Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-28T21:02:36.831543745Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T21:02:38.097211998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:40.250349428Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
ts=2026-04-28T21:02:42.619701262Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:02:43.097694217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:43.667321927Z caller=checkpoint.go:372 msg="writing series" size="33 kB"
level=debug ts=2026-04-28T21:02:44.037846909Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:02:45.557649292Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59882"
level=debug ts=2026-04-28T21:02:47.085692362Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-28T21:02:48.076578259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:02:48.088297549Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:02:48.097525935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:50.502870758Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T21:02:53.097893495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:02:53.920765333Z caller=checkpoint.go:372 msg="writing series" size="32 kB"
level=debug ts=2026-04-28T21:02:57.338640566Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
level=debug ts=2026-04-28T21:02:57.338665203Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=21
level=info ts=2026-04-28T21:02:58.079737895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:02:58.080285717Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:02:58.097401724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:02:58.097423374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:02:58.097430845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:02:58.097430546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:02:59.568161339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:02:59.577200791Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:03:00.760918Z caller=checkpoint.go:372 msg="writing series" size="8.0 MB"
level=debug ts=2026-04-28T21:03:00.76094233Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.0 MB" series=1
level=debug ts=2026-04-28T21:03:01.542044214Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:03:01.54414386Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T21:03:03.097740546Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:04.174108359Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T21:03:07.592127301Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T21:03:08.098055098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:11.008857226Z caller=checkpoint.go:372 msg="writing series" size="159 kB"
level=debug ts=2026-04-28T21:03:12.051735111Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:03:12.621912217Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:03:13.09809125Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:14.426699971Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
ts=2026-04-28T21:03:15.560321866Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37772"
level=info ts=2026-04-28T21:03:17.216179967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:03:17.225388954Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:03:17.845281194Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-04-28T21:03:18.097968866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:21.262667105Z caller=checkpoint.go:372 msg="writing series" size="75 kB"
level=warn ts=2026-04-28T21:03:23.097514914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:24.67973297Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=debug ts=2026-04-28T21:03:28.09759749Z caller=checkpoint.go:384 msg="flushing series" totalSize="431 kB" series=7
level=info ts=2026-04-28T21:03:28.097629307Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:03:28.097640656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:28.097650893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:03:28.097659282Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:03:28.151158165Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-28T21:03:28.156129807Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.058819595s
level=warn ts=2026-04-28T21:03:33.097817998Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:03:38.097600405Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:03:38.864032474Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:03:41.373798027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:03:42.623819882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:03:43.097681412Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:03:45.689590383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.5 MB" avg_comp="2.7 MB" total_uncomp="47 MB" avg_uncomp="23 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:03:45.69915222Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:03:48.097342525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:03:53.09796557Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:03:57.57221461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="119 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:03:57.584532531Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:03:58.078945004Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:03:58.078958001Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T21:03:58.097061249Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:03:58.09707529Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:03:58.097101147Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:03:58.097110669Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:03:58.097187918Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-28T21:03:58.097195604Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=warn ts=2026-04-28T21:03:58.09719464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:03:58.09721349Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.427µs success=true
level=debug ts=2026-04-28T21:03:58.097394446Z caller=checkpoint.go:372 msg="writing series" size="2.3 kB"
level=debug ts=2026-04-28T21:04:01.515977683Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T21:04:03.098008562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:03.757374616Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:04:04.948029713Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-04-28T21:04:04.948062174Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=3
level=warn ts=2026-04-28T21:04:08.097209128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:08.351253251Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=debug ts=2026-04-28T21:04:11.769179369Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
ts=2026-04-28T21:04:12.625917271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:04:13.097694521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:15.186506672Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-04-28T21:04:18.097533106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:18.6044693Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-04-28T21:04:22.022248068Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-04-28T21:04:23.103657775Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:23.354000977Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:04:25.442836826Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-28T21:04:25.442862661Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=6
level=info ts=2026-04-28T21:04:28.102355483Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:04:28.102407704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:04:28.102418856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:04:28.102458985Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:28.282894137Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:04:28.862408468Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-04-28T21:04:32.276231397Z caller=checkpoint.go:372 msg="writing series" size="411 kB"
level=warn ts=2026-04-28T21:04:33.097863205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:35.693351055Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=warn ts=2026-04-28T21:04:38.098164255Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:04:38.952468703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:04:38.971005828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:04:39.110264444Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
level=debug ts=2026-04-28T21:04:42.530841675Z caller=checkpoint.go:372 msg="writing series" size="167 kB"
ts=2026-04-28T21:04:42.627879626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:04:43.09737727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:45.946404741Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T21:04:48.098319367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:49.3642073Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=debug ts=2026-04-28T21:04:49.364234194Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-04-28T21:04:52.781962563Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-04-28T21:04:53.097307208Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:56.19949823Z caller=checkpoint.go:372 msg="writing series" size="679 kB"
level=info ts=2026-04-28T21:04:57.382094669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="142 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:04:57.39424532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:04:58.079243328Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:04:58.079890158Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:04:58.097899807Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:04:58.097907497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:04:58.097928244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:04:58.097941003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:04:59.617390637Z caller=checkpoint.go:372 msg="writing series" size="353 kB"
level=debug ts=2026-04-28T21:04:59.61742077Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=debug ts=2026-04-28T21:05:00.522263869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:05:01.657877993Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:05:03.034198746Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:05:03.097263741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:06.452849657Z caller=checkpoint.go:372 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T21:05:08.097899847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:09.87079927Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:05:12.630542783Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:05:13.09716594Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:13.288013412Z caller=checkpoint.go:372 msg="writing series" size="100 kB"
ts=2026-04-28T21:05:14.063691123Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35964"
ts=2026-04-28T21:05:15.569185352Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57200"
level=debug ts=2026-04-28T21:05:16.705239655Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-28T21:05:18.097514631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:20.123210752Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=warn ts=2026-04-28T21:05:23.097317543Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:23.542497866Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=debug ts=2026-04-28T21:05:23.542526613Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-04-28T21:05:26.958497386Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:05:28.097080076Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:05:28.097117829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:05:28.09713026Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:05:28.097147174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:30.376439385Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T21:05:33.097952448Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:33.794186267Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-28T21:05:35.372497319Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:05:37.211412593Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:05:38.097218988Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:05:38.650144618Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:05:38.664091259Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:05:38.664115899Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:05:38.664145079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:05:38.672936282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:05:40.629230864Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
ts=2026-04-28T21:05:42.632929568Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:05:43.097864136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:44.046823935Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T21:05:47.465267288Z caller=checkpoint.go:372 msg="writing series" size="367 kB"
level=warn ts=2026-04-28T21:05:48.097828317Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:50.805732957Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:05:50.883208583Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-04-28T21:05:51.045603229Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:05:51.055779541Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:05:53.097655769Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:05:54.300787143Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:05:55.574797384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.9 MB" total_uncomp="165 MB" avg_uncomp="24 MB" full=7 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:05:55.582580616Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:05:57.71776098Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-04-28T21:05:58.079168499Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:05:58.079670512Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:05:58.098666335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:05:58.098694038Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:05:58.098704594Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:05:58.098700177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:01.136868077Z caller=checkpoint.go:372 msg="writing series" size="60 kB"
level=warn ts=2026-04-28T21:06:03.097831072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:04.553893992Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-04-28T21:06:07.972139871Z caller=checkpoint.go:372 msg="writing series" size="657 kB"
level=debug ts=2026-04-28T21:06:07.972163686Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=13
level=warn ts=2026-04-28T21:06:08.097815256Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:08.846457227Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:06:10.142912673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34058"
level=debug ts=2026-04-28T21:06:11.389141822Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-28T21:06:12.634643723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:06:13.097739096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:14.806318739Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
ts=2026-04-28T21:06:15.572735809Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49472"
level=info ts=2026-04-28T21:06:17.200117547Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:06:17.211995135Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:06:18.097838232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:18.224340522Z caller=checkpoint.go:372 msg="writing series" size="410 kB"
level=debug ts=2026-04-28T21:06:21.641936689Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-04-28T21:06:23.097904039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:25.060033157Z caller=checkpoint.go:372 msg="writing series" size="394 kB"
level=debug ts=2026-04-28T21:06:25.060056755Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-28T21:06:26.547661796Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:06:26.550403067Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T21:06:28.097655326Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:06:28.097663578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:28.097681322Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:06:28.097690377Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:06:28.477695923Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-04-28T21:06:31.895271553Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=warn ts=2026-04-28T21:06:33.097662551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:35.312879675Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=debug ts=2026-04-28T21:06:35.512886075Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:06:38.097238145Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:38.730533289Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-04-28T21:06:40.144961366Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50690"
level=debug ts=2026-04-28T21:06:42.148293754Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:06:42.640695019Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:06:43.098033453Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:06:44.070029329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56952"
level=debug ts=2026-04-28T21:06:45.566461471Z caller=checkpoint.go:372 msg="writing series" size="400 kB"
level=warn ts=2026-04-28T21:06:48.097290653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:48.983832892Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-04-28T21:06:48.983855509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-04-28T21:06:52.402137361Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:06:52.726142524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.9 MB" total_uncomp="189 MB" avg_uncomp="24 MB" full=8 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:06:52.738687526Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:06:53.097367553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:55.81962701Z caller=checkpoint.go:372 msg="writing series" size="99 kB"
level=info ts=2026-04-28T21:06:58.079007013Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:06:58.080038212Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:06:58.097156992Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:06:58.097181122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:06:58.097190986Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:06:58.097218322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:06:59.237890262Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-04-28T21:07:02.654944873Z caller=checkpoint.go:372 msg="writing series" size="45 kB"
level=warn ts=2026-04-28T21:07:03.097422725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:06.072645578Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=debug ts=2026-04-28T21:07:06.602298114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:07:08.097294725Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:07:09.132840902Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:07:09.143015829Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:07:09.49114434Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-28T21:07:09.491166298Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
ts=2026-04-28T21:07:12.642366875Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:07:12.908096968Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T21:07:13.097542026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:16.32607666Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T21:07:17.825789075Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:07:18.097235043Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:19.743846915Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:07:23.097644261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:23.160870549Z caller=checkpoint.go:372 msg="writing series" size="8.1 kB"
level=debug ts=2026-04-28T21:07:26.579208195Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-04-28T21:07:28.097284311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:07:28.097312211Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:07:28.097321945Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:07:28.097365516Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:29.99693317Z caller=checkpoint.go:372 msg="writing series" size="158 kB"
level=warn ts=2026-04-28T21:07:33.097298253Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:33.415279522Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-28T21:07:36.832079735Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T21:07:38.09731371Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:07:40.152852465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51700"
level=debug ts=2026-04-28T21:07:40.249758914Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=debug ts=2026-04-28T21:07:41.655785903Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:07:42.648786884Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:07:43.097119268Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:43.668380499Z caller=checkpoint.go:372 msg="writing series" size="34 kB"
ts=2026-04-28T21:07:45.587847299Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36370"
level=debug ts=2026-04-28T21:07:47.085806706Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-28T21:07:48.097199328Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:50.503516723Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-04-28T21:07:51.21086779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=9 total_comp="26 MB" avg_comp="2.9 MB" total_uncomp="213 MB" avg_uncomp="24 MB" full=9 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:07:51.221803047Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 9, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:07:53.098125428Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:07:53.921092521Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-04-28T21:07:57.339851454Z caller=checkpoint.go:372 msg="writing series" size="833 kB"
level=debug ts=2026-04-28T21:07:57.339881174Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=14
level=info ts=2026-04-28T21:07:58.078976735Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:07:58.079939464Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:07:58.097658983Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:07:58.097682467Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:07:58.097676378Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:07:58.097691806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:07:58.363739936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:07:58.376136814Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:08:00.777745949Z caller=checkpoint.go:372 msg="writing series" size="8.6 MB"
level=debug ts=2026-04-28T21:08:00.777772261Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.6 MB" series=1
level=warn ts=2026-04-28T21:08:03.097176944Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:08:04.174051624Z caller=checkpoint.go:372 msg="writing series" size="92 kB"
level=debug ts=2026-04-28T21:08:07.591860559Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T21:08:08.097805099Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:08:11.009737158Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
ts=2026-04-28T21:08:12.654059202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:08:13.097141171Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:08:13.362460679Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:08:14.076887291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34426"
level=debug ts=2026-04-28T21:08:14.427404198Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-04-28T21:08:15.083826984Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:08:17.844972125Z caller=checkpoint.go:372 msg="writing series" size="343 kB"
level=warn ts=2026-04-28T21:08:18.097365476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:08:21.262142048Z caller=checkpoint.go:372 msg="writing series" size="111 kB"
level=warn ts=2026-04-28T21:08:23.098029303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:08:24.680064962Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-04-28T21:08:28.097377392Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:08:28.097384712Z caller=checkpoint.go:384 msg="flushing series" totalSize="929 kB" series=7
level=debug ts=2026-04-28T21:08:28.097404266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:08:28.097414398Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:08:28.097432743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:08:28.104762391Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-28T21:08:28.105898192Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008552182s
level=warn ts=2026-04-28T21:08:33.097975391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:08:38.097137427Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:08:40.157286904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48402"
level=info ts=2026-04-28T21:08:41.054392593Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=10 total_comp="29 MB" avg_comp="2.9 MB" total_uncomp="236 MB" avg_uncomp="24 MB" full=10 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:08:41.06660442Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 10, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:08:42.656571372Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:08:43.098183813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:08:44.078271595Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:42914"
level=debug ts=2026-04-28T21:08:45.428821859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:08:48.097318673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:08:48.303993705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:08:48.312556982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:08:53.097313535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:08:58.078993461Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:08:58.079095891Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:08:58.097534098Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-28T21:08:58.097536237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:08:58.097541518Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:08:58.097564894Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:08:58.097572747Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:08:58.098005899Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-28T21:08:58.098017977Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:08:58.09803882Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.74µs success=true
level=debug ts=2026-04-28T21:08:58.098260966Z caller=checkpoint.go:372 msg="writing series" size="2.5 kB"
level=debug ts=2026-04-28T21:09:01.473648499Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T21:09:03.09804077Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:04.876917321Z caller=checkpoint.go:372 msg="writing series" size="30 MB"
level=debug ts=2026-04-28T21:09:04.876946348Z caller=checkpoint.go:384 msg="flushing series" totalSize="30 MB" series=3
level=warn ts=2026-04-28T21:09:08.097507701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:08.22407398Z caller=checkpoint.go:372 msg="writing series" size="373 kB"
level=debug ts=2026-04-28T21:09:09.561785033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:09:11.599121676Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
ts=2026-04-28T21:09:12.658650171Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:09:13.097414214Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:14.973944019Z caller=checkpoint.go:372 msg="writing series" size="172 kB"
level=debug ts=2026-04-28T21:09:15.716219029Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:09:18.097194301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:18.348819248Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-04-28T21:09:21.723913649Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-04-28T21:09:23.097951657Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:25.101383255Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-28T21:09:25.101405749Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=6
level=info ts=2026-04-28T21:09:28.097868472Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:09:28.097889906Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:09:28.097897641Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:09:28.097920156Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:28.474081759Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-04-28T21:09:31.849715076Z caller=checkpoint.go:372 msg="writing series" size="629 kB"
level=warn ts=2026-04-28T21:09:33.098241975Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:09:34.966264154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:09:34.975586755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:09:35.223354082Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=warn ts=2026-04-28T21:09:38.097211604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:38.598548915Z caller=checkpoint.go:372 msg="writing series" size="120 kB"
level=debug ts=2026-04-28T21:09:38.598573231Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=info ts=2026-04-28T21:09:40.960638736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:09:40.970774327Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:09:40.970801996Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:09:40.970837088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:09:40.979262882Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:09:41.395286731Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:09:41.973960996Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
ts=2026-04-28T21:09:42.661669903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:09:43.097323787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:45.350766216Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T21:09:48.097664464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:48.724994597Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T21:09:48.725020973Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-04-28T21:09:52.099283296Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T21:09:53.097713382Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:09:55.474541499Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T21:09:55.474563635Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-28T21:09:56.199899946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:09:56.213489773Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:09:58.079034761Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:09:58.079099165Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T21:09:58.098095678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:09:58.098086422Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:09:58.098119939Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:09:58.098130789Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:09:58.849189997Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-04-28T21:10:02.22424556Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:10:03.097271366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:05.599505781Z caller=checkpoint.go:372 msg="writing series" size="508 kB"
level=debug ts=2026-04-28T21:10:05.599534887Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-28T21:10:08.097240019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:08.974077371Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:10:10.163680633Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58734"
level=debug ts=2026-04-28T21:10:10.984788002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:10:12.348568054Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=debug ts=2026-04-28T21:10:12.48426231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:10:12.664335859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:10:13.097371911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:10:14.085445667Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48898"
level=debug ts=2026-04-28T21:10:15.723473285Z caller=checkpoint.go:372 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-28T21:10:18.098062993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:10:18.129268299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:10:18.145817946Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:10:19.098732636Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=debug ts=2026-04-28T21:10:22.473832481Z caller=checkpoint.go:372 msg="writing series" size="697 kB"
level=debug ts=2026-04-28T21:10:22.473860107Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-28T21:10:23.097164222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:25.848714475Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:10:28.098364924Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:10:28.098397993Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:10:28.098408666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:10:28.098436318Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:29.224215299Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-28T21:10:32.598766263Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T21:10:33.0975107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:10:34.070540384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="69 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:10:34.082770172Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:10:35.974086687Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=warn ts=2026-04-28T21:10:38.097180841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:39.349168907Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
level=debug ts=2026-04-28T21:10:40.885609521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:10:40.885681735Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:10:40.893959555Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
ts=2026-04-28T21:10:42.66694266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:10:42.724270972Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T21:10:43.097575329Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:10:45.599954242Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55774"
level=debug ts=2026-04-28T21:10:46.099112603Z caller=checkpoint.go:372 msg="writing series" size="310 kB"
level=debug ts=2026-04-28T21:10:46.184990035Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:10:46.188207624Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T21:10:48.09735521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:49.473337411Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-04-28T21:10:52.529999057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:10:52.546957113Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:10:52.848776393Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=warn ts=2026-04-28T21:10:53.097648026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:10:56.223329583Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-04-28T21:10:58.079154171Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:10:58.079559311Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:10:58.097777191Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:10:58.097799604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:10:58.097809396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:10:58.097807402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:10:59.191028172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=11 total_comp="32 MB" avg_comp="2.9 MB" total_uncomp="260 MB" avg_uncomp="24 MB" full=11 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:10:59.200540883Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 11, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:10:59.598830869Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=debug ts=2026-04-28T21:11:02.974118464Z caller=checkpoint.go:372 msg="writing series" size="522 kB"
level=debug ts=2026-04-28T21:11:02.974144878Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=12
level=warn ts=2026-04-28T21:11:03.097172357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:06.349322393Z caller=checkpoint.go:372 msg="writing series" size="526 kB"
level=debug ts=2026-04-28T21:11:07.378252997Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:11:08.097361124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:09.723513037Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-28T21:11:10.16832467Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33180"
ts=2026-04-28T21:11:12.668602854Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:11:13.097391844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:13.098466129Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
ts=2026-04-28T21:11:14.089599572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36060"
ts=2026-04-28T21:11:15.603672752Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40344"
level=debug ts=2026-04-28T21:11:16.473795009Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=debug ts=2026-04-28T21:11:16.473817651Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=4
level=warn ts=2026-04-28T21:11:18.097659595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:11:18.841695122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:11:18.85648676Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:11:19.849250766Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:11:23.097751578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:23.224311621Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=debug ts=2026-04-28T21:11:26.59917493Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:11:27.898990057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:11:27.908489446Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:11:28.098067651Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:11:28.098066181Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:28.098092187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:11:28.098099858Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:11:29.973690672Z caller=checkpoint.go:372 msg="writing series" size="434 kB"
level=debug ts=2026-04-28T21:11:29.973717478Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-28T21:11:33.098239701Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:33.349086561Z caller=checkpoint.go:372 msg="writing series" size="239 kB"
level=debug ts=2026-04-28T21:11:36.724019633Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T21:11:38.097385137Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:40.099149992Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:11:40.257428755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:11:42.474822959Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:11:42.670515044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:11:43.097748431Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:43.473456979Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-04-28T21:11:45.606213414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60054"
level=info ts=2026-04-28T21:11:45.862952024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=12 total_comp="35 MB" avg_comp="2.9 MB" total_uncomp="284 MB" avg_uncomp="24 MB" full=12 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:11:45.870775923Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 12, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:11:46.848667246Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=warn ts=2026-04-28T21:11:48.098071366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:50.223724228Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-04-28T21:11:50.223746794Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-28T21:11:53.098142569Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:11:53.598884967Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-04-28T21:11:56.973952719Z caller=checkpoint.go:372 msg="writing series" size="105 kB"
level=info ts=2026-04-28T21:11:58.079827563Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:11:58.080933643Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:11:58.097460677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:11:58.097486072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:11:58.097497095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:11:58.097508325Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:11:59.523965655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:11:59.535289909Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:12:00.348861471Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=warn ts=2026-04-28T21:12:03.097845946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:03.723889052Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=debug ts=2026-04-28T21:12:07.099218558Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=warn ts=2026-04-28T21:12:08.097882233Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:10.475980894Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-04-28T21:12:10.476001979Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.4 MB" series=6
ts=2026-04-28T21:12:12.671570044Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:12:13.097704642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:13.848847739Z caller=checkpoint.go:372 msg="writing series" size="128 kB"
level=debug ts=2026-04-28T21:12:17.043558217Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:12:17.224382509Z caller=checkpoint.go:372 msg="writing series" size="192 kB"
level=info ts=2026-04-28T21:12:18.01576144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:12:18.024735272Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:12:18.024764057Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:12:18.024795652Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:12:18.032059073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:12:18.097744152Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:20.598767795Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:12:23.097972688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:23.97359081Z caller=checkpoint.go:372 msg="writing series" size="9.0 kB"
level=debug ts=2026-04-28T21:12:27.349406486Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=info ts=2026-04-28T21:12:28.09767749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:12:28.097700659Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:28.097717106Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:12:28.097728044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:12:30.723777924Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-04-28T21:12:33.09814419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:12:33.868943923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="308 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:12:33.876607519Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:12:34.098314711Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-28T21:12:35.067031125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:12:36.775335215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:12:36.785292409Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:12:37.473316921Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T21:12:38.097143408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:40.84866753Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-04-28T21:12:42.673897566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:12:43.097392148Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:12:44.097127208Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48676"
level=debug ts=2026-04-28T21:12:44.224079529Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-04-28T21:12:47.598502569Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-28T21:12:48.097965184Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:50.974165801Z caller=checkpoint.go:372 msg="writing series" size="223 kB"
level=debug ts=2026-04-28T21:12:50.974191378Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-04-28T21:12:52.791566734Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:12:53.050189426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:12:53.061503766Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:12:53.098132003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:12:54.349345061Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-28T21:12:57.724603255Z caller=checkpoint.go:372 msg="writing series" size="898 kB"
level=info ts=2026-04-28T21:12:58.079041393Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:12:58.080414879Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:12:58.097732376Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:12:58.097757111Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:12:58.097767242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:12:58.097783673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:12:59.091483133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:12:59.099792425Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:13:01.111064616Z caller=checkpoint.go:372 msg="writing series" size="9.4 MB"
level=debug ts=2026-04-28T21:13:01.111229946Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=3
level=warn ts=2026-04-28T21:13:03.097389356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:13:04.473468224Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=debug ts=2026-04-28T21:13:07.849209339Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T21:13:08.097281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:13:11.223707295Z caller=checkpoint.go:372 msg="writing series" size="218 kB"
ts=2026-04-28T21:13:12.67659629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:13:13.098174653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:13:14.598863926Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=debug ts=2026-04-28T21:13:17.974354674Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-04-28T21:13:18.097402663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:13:20.495834403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=13 total_comp="38 MB" avg_comp="2.9 MB" total_uncomp="308 MB" avg_uncomp="24 MB" full=13 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:13:20.504501562Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 13, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:13:21.022937859Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:13:21.34844731Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=warn ts=2026-04-28T21:13:23.098107293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:13:23.828699536Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:13:24.723628568Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-04-28T21:13:28.097171282Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:13:28.097181155Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:13:28.09721002Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:13:28.097219191Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:13:28.098252079Z caller=checkpoint.go:384 msg="flushing series" totalSize="952 kB" series=7
level=info ts=2026-04-28T21:13:28.169197676Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-28T21:13:28.171383998Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.073182998s
level=warn ts=2026-04-28T21:13:33.097197386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:13:38.097428918Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:13:40.396511812Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:13:40.405004483Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
ts=2026-04-28T21:13:42.678378207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:13:43.097315604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:13:45.613208762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49154"
level=info ts=2026-04-28T21:13:46.905161364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:13:46.912565219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:13:48.097582433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:13:52.731523413Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:13:53.097673525Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:13:58.079477261Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:13:58.080168089Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T21:13:58.097137627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:13:58.097136746Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:13:58.097170065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:13:58.097190032Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:13:58.097197572Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:13:58.097284234Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-28T21:13:58.097292809Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:13:58.097309792Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.327µs success=true
level=debug ts=2026-04-28T21:13:58.097473796Z caller=checkpoint.go:372 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-28T21:14:01.431060885Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T21:14:03.097394026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:04.801481998Z caller=checkpoint.go:372 msg="writing series" size="40 MB"
level=debug ts=2026-04-28T21:14:04.801518758Z caller=checkpoint.go:384 msg="flushing series" totalSize="40 MB" series=3
level=warn ts=2026-04-28T21:14:08.097252819Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:08.098798276Z caller=checkpoint.go:372 msg="writing series" size="608 kB"
ts=2026-04-28T21:14:10.178803687Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60402"
level=debug ts=2026-04-28T21:14:11.431215989Z caller=checkpoint.go:372 msg="writing series" size="361 kB"
ts=2026-04-28T21:14:12.680043748Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:14:12.955531788Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:14:13.097254857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:14.764867793Z caller=checkpoint.go:372 msg="writing series" size="319 kB"
level=debug ts=2026-04-28T21:14:14.76489099Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=info ts=2026-04-28T21:14:16.33697058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:14:16.346562262Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:14:18.097817172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:18.097904769Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-04-28T21:14:21.431597247Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=debug ts=2026-04-28T21:14:22.883272673Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:14:22.989672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:14:22.998786544Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:14:23.097669469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:24.767429693Z caller=checkpoint.go:372 msg="writing series" size="3.1 MB"
level=debug ts=2026-04-28T21:14:24.767454168Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.3 MB" series=3
level=info ts=2026-04-28T21:14:28.097261998Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:14:28.097300968Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:14:28.097297818Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:14:28.097312668Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:14:28.09857443Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-04-28T21:14:31.431190388Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:14:33.097330898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:34.764514766Z caller=checkpoint.go:372 msg="writing series" size="560 kB"
level=info ts=2026-04-28T21:14:34.82746624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:14:34.841170184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:14:38.097459541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:38.097506682Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:14:41.431717458Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=debug ts=2026-04-28T21:14:41.431739886Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-04-28T21:14:42.681893411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:14:43.098175219Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:14:44.105610845Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40362"
level=debug ts=2026-04-28T21:14:44.764700332Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
ts=2026-04-28T21:14:45.616776528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:42514"
level=warn ts=2026-04-28T21:14:48.097556399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:48.097574428Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-28T21:14:51.433447245Z caller=checkpoint.go:372 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-28T21:14:51.433468574Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-04-28T21:14:53.097933946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:54.764835503Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-04-28T21:14:56.187241248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:14:56.187290435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-28T21:14:56.198817167Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:14:58.079098662Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:14:58.079598201Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:14:58.097242663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:14:58.097249947Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:14:58.097264872Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:14:58.097272544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:14:58.100357083Z caller=checkpoint.go:372 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-28T21:14:58.100368475Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-04-28T21:15:01.432107932Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=warn ts=2026-04-28T21:15:03.097642242Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:04.764267364Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:15:05.693944784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:15:05.703939786Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:15:08.098107936Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:08.098537672Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:15:08.098554324Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-04-28T21:15:11.116145542Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:15:11.117969235Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:15:11.431955574Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:15:12.683501482Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:15:13.097915727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:14.764680808Z caller=checkpoint.go:372 msg="writing series" size="313 kB"
level=info ts=2026-04-28T21:15:15.272065407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=14 total_comp="40 MB" avg_comp="2.9 MB" total_uncomp="332 MB" avg_uncomp="24 MB" full=14 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:15:15.280007969Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 14, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:15:15.883954768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:15:15.891381002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:15:18.098066856Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:18.098069683Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-28T21:15:21.431317901Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=warn ts=2026-04-28T21:15:23.097676313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:24.765411996Z caller=checkpoint.go:372 msg="writing series" size="849 kB"
level=debug ts=2026-04-28T21:15:24.765435168Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=5
level=info ts=2026-04-28T21:15:28.097936375Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:15:28.097960375Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:15:28.097968362Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:15:28.098011993Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:28.098011646Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:15:28.472356602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:15:28.484318413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:15:31.431222088Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-28T21:15:31.634951678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:15:33.098009096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:34.765033847Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T21:15:38.097743846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:38.097895595Z caller=checkpoint.go:372 msg="writing series" size="140 kB"
level=debug ts=2026-04-28T21:15:41.431599303Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
ts=2026-04-28T21:15:42.685463009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:15:43.098097562Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:44.764455385Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T21:15:48.097236289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:48.098495908Z caller=checkpoint.go:372 msg="writing series" size="408 kB"
level=debug ts=2026-04-28T21:15:48.09851225Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-04-28T21:15:51.431939077Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-04-28T21:15:52.225146209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:15:52.732562537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:15:52.741368707Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:15:53.097185187Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:15:54.764351336Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=debug ts=2026-04-28T21:15:56.40830048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:15:58.079198641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:15:58.079183813Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:15:58.080459835Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T21:15:58.080506001Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974900 n=1
level=debug ts=2026-04-28T21:15:58.080546948Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974900 n=1
level=info ts=2026-04-28T21:15:58.097856586Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:15:58.09788641Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:15:58.097894448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:15:58.097922412Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=warn ts=2026-04-28T21:15:58.097931831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:01.431203206Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:16:03.097155634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:16:03.509939202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:16:03.521500907Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:16:04.765116011Z caller=checkpoint.go:372 msg="writing series" size="489 kB"
level=info ts=2026-04-28T21:16:06.793586763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:16:06.801186779Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:16:06.801211855Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:16:06.801243879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:16:06.808167207Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:16:08.097166026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:08.099159658Z caller=checkpoint.go:372 msg="writing series" size="553 kB"
level=debug ts=2026-04-28T21:16:08.099174741Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=debug ts=2026-04-28T21:16:11.431509367Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-28T21:16:12.687834928Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:16:13.098145032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:14.764775805Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=warn ts=2026-04-28T21:16:18.097319773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:18.098882233Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:16:19.259408095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:16:19.271011321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:16:21.113689537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:16:21.125544137Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:16:21.431580267Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-28T21:16:22.715305801Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:16:23.097169689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:24.765359853Z caller=checkpoint.go:372 msg="writing series" size="331 kB"
level=debug ts=2026-04-28T21:16:24.76538824Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T21:16:28.097787687Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:16:28.097820023Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:16:28.097827775Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:16:28.097844387Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:28.097937094Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-04-28T21:16:31.431927227Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=warn ts=2026-04-28T21:16:33.097489469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:34.764405346Z caller=checkpoint.go:372 msg="writing series" size="300 kB"
level=debug ts=2026-04-28T21:16:38.097984256Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T21:16:38.098015429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:41.431349823Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:16:41.431370217Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=debug ts=2026-04-28T21:16:42.51720486Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:16:42.690135644Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:16:43.098075154Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:16:44.114517971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48386"
level=debug ts=2026-04-28T21:16:44.764775315Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=info ts=2026-04-28T21:16:47.73359871Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:16:47.742955432Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:16:48.09740541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:48.097776485Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T21:16:49.511470426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:16:49.520652845Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:16:51.431923318Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=debug ts=2026-04-28T21:16:51.516501321Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:16:53.097442322Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:54.764625915Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:16:58.079019179Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:16:58.079425057Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:16:58.097233257Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:16:58.097251796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:16:58.097259314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:16:58.097261609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:16:58.098459331Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-04-28T21:16:59.142969624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:16:59.15178077Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:16:59.151803559Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:16:59.15183555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:16:59.161427893Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:17:01.431225851Z caller=checkpoint.go:372 msg="writing series" size="93 kB"
level=warn ts=2026-04-28T21:17:03.097349941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:04.764517707Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:17:08.097367846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:08.097558476Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=debug ts=2026-04-28T21:17:11.43896831Z caller=checkpoint.go:372 msg="writing series" size="6.2 MB"
level=debug ts=2026-04-28T21:17:11.438991889Z caller=checkpoint.go:384 msg="flushing series" totalSize="7.1 MB" series=9
level=info ts=2026-04-28T21:17:11.79550473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:17:11.806568506Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T21:17:12.691711163Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:17:13.097453194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:17:14.624020507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:17:14.631716933Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:17:14.765214475Z caller=checkpoint.go:372 msg="writing series" size="183 kB"
level=warn ts=2026-04-28T21:17:18.097224045Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:18.098343431Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-04-28T21:17:21.43186004Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:17:23.098192275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:24.459221451Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:17:24.459307441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="534 kB" avg_comp="534 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:17:24.465585871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:17:24.764604904Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T21:17:28.09735698Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:17:28.097384266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:17:28.097391375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:17:28.0974207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:28.098628131Z caller=checkpoint.go:372 msg="writing series" size="210 kB"
level=debug ts=2026-04-28T21:17:31.43130801Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
level=warn ts=2026-04-28T21:17:33.097688883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:17:34.44713322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:17:34.457037843Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:17:34.765045685Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:17:37.572930768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:17:37.578233557Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:17:38.097340899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:38.098679805Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T21:17:39.876959449Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:17:41.242916094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:17:41.251130233Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:17:41.43138121Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-04-28T21:17:42.693885292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:17:43.09756999Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:44.764768902Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-04-28T21:17:48.097363067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:48.098495279Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-28T21:17:49.728741213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:17:49.737687668Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:17:51.431339034Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=debug ts=2026-04-28T21:17:51.431362571Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-04-28T21:17:53.097561261Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:54.765113206Z caller=checkpoint.go:372 msg="writing series" size="39 kB"
level=debug ts=2026-04-28T21:17:56.855843294Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:17:58.079107514Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:17:58.079120818Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:17:58.097081318Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:17:58.097106341Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:17:58.097114088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:17:58.097182699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:17:58.099236093Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-28T21:17:58.099252642Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=2
level=info ts=2026-04-28T21:17:59.327908457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:17:59.337542904Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:18:01.441086881Z caller=checkpoint.go:372 msg="writing series" size="10 MB"
level=debug ts=2026-04-28T21:18:01.441109708Z caller=checkpoint.go:384 msg="flushing series" totalSize="10 MB" series=1
level=warn ts=2026-04-28T21:18:03.097322113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:18:04.764732906Z caller=checkpoint.go:372 msg="writing series" size="154 kB"
level=debug ts=2026-04-28T21:18:08.097584861Z caller=checkpoint.go:372 msg="writing series" size="515 B"
level=warn ts=2026-04-28T21:18:08.097586091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:18:10.207338042Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34804"
level=debug ts=2026-04-28T21:18:11.431063052Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=info ts=2026-04-28T21:18:12.13665661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:18:12.143479229Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:18:12.700440167Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:18:13.098101893Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:18:14.764558366Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:18:15.629077308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54308"
level=debug ts=2026-04-28T21:18:16.894697786Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:18:18.097175521Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:18:18.098938486Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:18:18.098952868Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-04-28T21:18:21.4313185Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=warn ts=2026-04-28T21:18:23.097560977Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:18:24.765061071Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-04-28T21:18:26.992590138Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:18:28.097778238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:18:28.097779805Z caller=checkpoint.go:384 msg="flushing series" totalSize="212 kB" series=2
level=debug ts=2026-04-28T21:18:28.097803947Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:18:28.09781155Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:18:28.097835808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:28.125177341Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-28T21:18:28.128088801Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.030663811s
level=info ts=2026-04-28T21:18:31.723509789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:18:31.733736896Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:18:33.097333825Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:34.653685416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:18:34.662770656Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:18:38.097804069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:39.710008639Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=15 total_comp="43 MB" avg_comp="2.9 MB" total_uncomp="356 MB" avg_uncomp="24 MB" full=15 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:18:39.717152995Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 15, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:18:42.702813386Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:18:43.097981836Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:18:45.631576057Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60488"
level=warn ts=2026-04-28T21:18:48.097956107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:52.453495337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:18:52.464770893Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:18:52.464798985Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:18:52.464825865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:18:52.47172391Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:18:53.097466658Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:53.279132731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:18:53.2834652Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:18:56.310041808Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:18:56.310103488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="332 kB" avg_comp="332 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:18:56.314301927Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:18:58.079045938Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:18:58.080304102Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T21:18:58.097554781Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:18:58.097580298Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-28T21:18:58.097581809Z caller=checkpoint.go:712 msg="starting checkpoint"
level=debug ts=2026-04-28T21:18:58.097599333Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:18:58.097608649Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:18:58.097891087Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-28T21:18:58.097901696Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:18:58.097922044Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.014µs success=true
level=debug ts=2026-04-28T21:18:58.098111475Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-28T21:19:01.432450956Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T21:19:03.098197875Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:04.817608143Z caller=checkpoint.go:372 msg="writing series" size="47 MB"
level=debug ts=2026-04-28T21:19:04.817731029Z caller=checkpoint.go:384 msg="flushing series" totalSize="47 MB" series=3
level=warn ts=2026-04-28T21:19:08.097686817Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:08.099200127Z caller=checkpoint.go:372 msg="writing series" size="523 kB"
level=info ts=2026-04-28T21:19:08.480369158Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:19:08.489662081Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:19:09.956226992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:19:09.963858602Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:19:11.433223836Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:19:11.433244614Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=2
level=debug ts=2026-04-28T21:19:11.862502155Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:19:11.86427619Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:19:12.70480577Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T21:19:13.050672059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:19:13.058474514Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:19:13.097474744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:19:14.124233427Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:58820"
level=debug ts=2026-04-28T21:19:14.765166826Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=warn ts=2026-04-28T21:19:18.097817837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:18.099005613Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-04-28T21:19:21.432512023Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-28T21:19:23.097184949Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:24.768453357Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-04-28T21:19:24.768474098Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.9 MB" series=4
level=debug ts=2026-04-28T21:19:25.79270239Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:19:28.097925144Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:19:28.097955822Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:19:28.097965843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:19:28.097973824Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:28.09927152Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:19:29.970667583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:19:29.980298403Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:19:31.43251154Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:19:31.999991878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:19:32.010477151Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:19:33.097143773Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:19:34.271030416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:19:34.279181935Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:19:34.76593137Z caller=checkpoint.go:372 msg="writing series" size="751 kB"
level=debug ts=2026-04-28T21:19:34.765957996Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-28T21:19:38.097940252Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:38.098978911Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:19:41.431908808Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
ts=2026-04-28T21:19:42.706691919Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T21:19:42.996546513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:19:43.002808618Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:19:43.097640456Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:19:44.125285078Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43418"
level=debug ts=2026-04-28T21:19:44.765484764Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=info ts=2026-04-28T21:19:47.164518133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:19:47.170443674Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:19:48.097234655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:48.098371121Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-28T21:19:51.433683174Z caller=checkpoint.go:372 msg="writing series" size="1.9 MB"
level=debug ts=2026-04-28T21:19:51.433704161Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=5
level=warn ts=2026-04-28T21:19:53.097473278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:54.76510156Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-04-28T21:19:58.07929125Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:19:58.080655755Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:19:58.097885102Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:19:58.097889887Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:19:58.097904162Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:19:58.097912239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:19:58.100151752Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-28T21:19:58.10016517Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=debug ts=2026-04-28T21:20:00.432603989Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:20:01.432236665Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=debug ts=2026-04-28T21:20:02.398860601Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:20:03.098022866Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:04.765864758Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:20:08.097394275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:08.09922364Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:20:08.099238575Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-04-28T21:20:11.432061612Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:20:12.708580989Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T21:20:12.896150869Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:20:12.908064737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:20:13.097729004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:20:14.127873206Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:45514"
level=debug ts=2026-04-28T21:20:14.765944876Z caller=checkpoint.go:372 msg="writing series" size="216 kB"
level=warn ts=2026-04-28T21:20:18.097949129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:18.098996619Z caller=checkpoint.go:372 msg="writing series" size="4.8 kB"
level=info ts=2026-04-28T21:20:20.906447307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:20:20.915162229Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:20:21.431953713Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-04-28T21:20:22.994253907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:20:23.001565491Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:20:23.098066098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:24.766129599Z caller=checkpoint.go:372 msg="writing series" size="827 kB"
level=debug ts=2026-04-28T21:20:24.766151433Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T21:20:27.205193617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:20:27.213321842Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:20:28.097333128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:20:28.097362524Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:20:28.097371661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:20:28.097379424Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:28.098457058Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-04-28T21:20:31.101628515Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:20:31.432422673Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T21:20:33.098167863Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:34.765836717Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T21:20:38.098190331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:38.09829463Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-04-28T21:20:40.217822574Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37970"
level=debug ts=2026-04-28T21:20:41.432127805Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T21:20:41.440932423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:20:41.447818585Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:20:42.710537715Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:20:43.097801697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:44.765552914Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-28T21:20:45.486982758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:20:45.493083595Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:20:48.097739938Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:48.099122542Z caller=checkpoint.go:372 msg="writing series" size="323 kB"
level=debug ts=2026-04-28T21:20:48.099136317Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=debug ts=2026-04-28T21:20:48.74845026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:20:51.432205501Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=warn ts=2026-04-28T21:20:53.097721039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:54.765053542Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:20:56.026361774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:20:56.036490123Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:20:58.079066283Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:20:58.079517637Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:20:58.098087043Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:20:58.098107788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:20:58.0981287Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:20:58.098174603Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:20:58.098203212Z caller=checkpoint.go:372 msg="writing series" size="88 kB"
level=debug ts=2026-04-28T21:20:59.70109556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:21:01.431549829Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:21:03.097857513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:04.76541403Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-04-28T21:21:08.097994059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:08.099419137Z caller=checkpoint.go:372 msg="writing series" size="762 kB"
level=debug ts=2026-04-28T21:21:08.099435026Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-04-28T21:21:08.898507908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:21:08.909676338Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:21:10.220476665Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50630"
level=debug ts=2026-04-28T21:21:11.431668149Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=info ts=2026-04-28T21:21:12.37717661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:21:12.385839582Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
ts=2026-04-28T21:21:12.71247716Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:21:13.097331673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:14.76535974Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=warn ts=2026-04-28T21:21:18.097487128Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:18.098840233Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:21:19.781186009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:21:19.789675134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:21:21.431928039Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:21:23.097295957Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:21:23.171660884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:21:23.176508889Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:21:24.765460347Z caller=checkpoint.go:372 msg="writing series" size="396 kB"
level=debug ts=2026-04-28T21:21:24.765481596Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T21:21:28.097071505Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:21:28.097102413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:21:28.097111952Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:21:28.097141794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:28.098181808Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=debug ts=2026-04-28T21:21:31.432347418Z caller=checkpoint.go:372 msg="writing series" size="454 kB"
level=warn ts=2026-04-28T21:21:33.09740883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:33.984778851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:21:34.765835194Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-04-28T21:21:38.097811811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:38.098896178Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=info ts=2026-04-28T21:21:38.543585157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:21:38.551176535Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:21:38.606358666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:21:38.614057977Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:21:38.614077363Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:21:38.614099236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:21:38.621900846Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:21:41.431809507Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:21:41.431826951Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-04-28T21:21:42.713804183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:21:43.097929514Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:44.764841175Z caller=checkpoint.go:372 msg="writing series" size="3.0 kB"
ts=2026-04-28T21:21:45.643713838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:35720"
level=debug ts=2026-04-28T21:21:46.137654712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:21:48.097145604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:48.098487952Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T21:21:49.535176904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:21:49.544050786Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:21:51.431955505Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=warn ts=2026-04-28T21:21:53.097443982Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:54.765234361Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:21:55.862353631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:21:55.878240227Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=info ts=2026-04-28T21:21:58.079475896Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:21:58.079921285Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:21:58.097632462Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:21:58.097650595Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:21:58.097658845Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:21:58.097654028Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:21:58.098837242Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=debug ts=2026-04-28T21:22:01.432568729Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=warn ts=2026-04-28T21:22:03.09723368Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:04.174714228Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:22:04.765153303Z caller=checkpoint.go:372 msg="writing series" size="64 kB"
level=info ts=2026-04-28T21:22:05.726025929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=16 total_comp="46 MB" avg_comp="2.9 MB" total_uncomp="380 MB" avg_uncomp="24 MB" full=16 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:22:05.733783647Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 16, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:22:08.097877831Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:08.098929353Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=debug ts=2026-04-28T21:22:11.436707103Z caller=checkpoint.go:372 msg="writing series" size="7.7 MB"
level=debug ts=2026-04-28T21:22:11.436731131Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.7 MB" series=9
level=info ts=2026-04-28T21:22:12.650912261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:22:12.658316508Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T21:22:12.715431033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:22:13.09757419Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:14.765530464Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T21:22:18.098043147Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:18.098195597Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-04-28T21:22:20.729258511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:22:20.734019629Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:22:21.432212517Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T21:22:22.091559086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:22:22.096530293Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:22:23.09713853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:22:24.31173678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:22:24.319056861Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:22:24.765284654Z caller=checkpoint.go:372 msg="writing series" size="11 kB"
level=info ts=2026-04-28T21:22:28.098106032Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:22:28.098102075Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:28.098132692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:22:28.098142393Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:22:28.098254233Z caller=checkpoint.go:372 msg="writing series" size="266 kB"
level=debug ts=2026-04-28T21:22:31.432668603Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=warn ts=2026-04-28T21:22:33.097239131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:34.765566701Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T21:22:38.097468808Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:38.098518412Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T21:22:39.91945772Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:22:41.432548763Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-04-28T21:22:42.716804085Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:22:43.097451429Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:22:44.140061186Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56736"
level=debug ts=2026-04-28T21:22:44.765238839Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-04-28T21:22:44.814206632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:22:47.764410265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:22:47.77117853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:22:47.771199695Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:22:47.771231947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:22:47.778537835Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:22:48.098025169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:48.098119967Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-28T21:22:51.432317908Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=debug ts=2026-04-28T21:22:51.432340392Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-04-28T21:22:52.904427123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:22:52.912666752Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:22:53.097389666Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:54.765098315Z caller=checkpoint.go:372 msg="writing series" size="41 kB"
level=info ts=2026-04-28T21:22:58.079164438Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:22:58.079174954Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:22:58.097869525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:22:58.097875339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:22:58.097887895Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:22:58.097895386Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:22:58.10013635Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T21:22:58.100148757Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=2
level=info ts=2026-04-28T21:22:59.984836601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:22:59.994247869Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:23:01.442435007Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-04-28T21:23:01.442466756Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=1
level=warn ts=2026-04-28T21:23:03.098135888Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:23:03.29688271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:23:03.302588865Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:23:04.765429598Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T21:23:05.034152373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:23:05.043489728Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:23:05.794980522Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:23:08.097373231Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:23:08.09851599Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=debug ts=2026-04-28T21:23:11.432560042Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
ts=2026-04-28T21:23:12.717803053Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=info ts=2026-04-28T21:23:12.723460149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:23:12.731007256Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:23:13.097673839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:23:13.764176098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:23:13.771076069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:23:14.764993898Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:23:15.650773528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49982"
level=warn ts=2026-04-28T21:23:18.097716892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:23:18.099509259Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:23:18.099524847Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-04-28T21:23:19.325882791Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:23:19.327610077Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:23:21.43196975Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=warn ts=2026-04-28T21:23:23.09807523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:23:24.765908999Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=info ts=2026-04-28T21:23:27.416450211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:23:27.427415801Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:23:28.097218178Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:23:28.097253692Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:23:28.097264617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:23:28.097269552Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:23:28.098326811Z caller=checkpoint.go:384 msg="flushing series" totalSize="260 kB" series=2
level=info ts=2026-04-28T21:23:28.105572351Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-28T21:23:28.10778193Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009727872s
level=warn ts=2026-04-28T21:23:33.097832789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:23:35.232447911Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:23:38.097286899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:23:40.228107481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33960"
ts=2026-04-28T21:23:42.719090228Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:23:43.097456185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:23:44.144924329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:36376"
level=warn ts=2026-04-28T21:23:48.097824715Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:23:48.951904923Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:23:48.975498166Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:23:48.975538781Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:23:48.975578072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:23:48.987796868Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=info ts=2026-04-28T21:23:51.380226549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:23:51.388342199Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:23:53.097740023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:23:55.820439274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:23:55.825258712Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:23:55.82528391Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:23:55.825306478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:23:55.828968575Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:23:58.079139312Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:23:58.079523665Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:23:58.097586162Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:23:58.097638313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:23:58.097661122Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:23:58.097658273Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:23:58.097668582Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:23:58.097868641Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-28T21:23:58.097878225Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:23:58.097897931Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.737µs success=true
level=debug ts=2026-04-28T21:23:58.098150931Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-28T21:23:59.741573103Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:24:01.196865013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:24:01.205267536Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:24:01.431923878Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-28T21:24:01.884505801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:24:01.888604605Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:24:03.097837774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:24:03.90375361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:24:03.911276101Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:24:04.81029736Z caller=checkpoint.go:372 msg="writing series" size="51 MB"
level=debug ts=2026-04-28T21:24:04.810326947Z caller=checkpoint.go:384 msg="flushing series" totalSize="51 MB" series=3
level=info ts=2026-04-28T21:24:06.757836665Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:24:06.762379065Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:24:08.097246417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:08.098924714Z caller=checkpoint.go:372 msg="writing series" size="633 kB"
level=debug ts=2026-04-28T21:24:10.656870915Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:24:11.433393579Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:24:11.433415104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
ts=2026-04-28T21:24:12.720753109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:24:13.098086235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:14.765450081Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-04-28T21:24:14.964887112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=17 total_comp="49 MB" avg_comp="2.9 MB" total_uncomp="404 MB" avg_uncomp="24 MB" full=17 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:24:14.974050826Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 17, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:24:18.097661408Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:18.098819473Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-04-28T21:24:21.432288988Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:24:23.098119474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:24:24.449880271Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:24:24.459470925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:24:24.767950472Z caller=checkpoint.go:372 msg="writing series" size="3.6 MB"
level=debug ts=2026-04-28T21:24:24.767970205Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.0 MB" series=4
level=info ts=2026-04-28T21:24:28.097631908Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:24:28.097661108Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:24:28.097669146Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:24:28.097692479Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:28.099052961Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=debug ts=2026-04-28T21:24:31.432159148Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:24:32.430635849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:24:32.436055827Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:24:33.097558341Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:34.765262016Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=warn ts=2026-04-28T21:24:38.097573898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:38.098711397Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:24:38.87737859Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:24:40.232435452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42772"
level=debug ts=2026-04-28T21:24:41.431839905Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=debug ts=2026-04-28T21:24:41.431863261Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=debug ts=2026-04-28T21:24:42.610947071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:24:42.722581774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:24:43.097628084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:24:43.796856118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:24:43.803043528Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:24:44.262348337Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:24:44.269667379Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:24:44.765425984Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
ts=2026-04-28T21:24:45.658535505Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:32780"
level=warn ts=2026-04-28T21:24:48.098033922Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:48.098181918Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T21:24:48.733935112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:24:48.74174278Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:24:51.433212975Z caller=checkpoint.go:372 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-28T21:24:51.433243559Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=3
level=warn ts=2026-04-28T21:24:53.097250573Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:54.765875546Z caller=checkpoint.go:372 msg="writing series" size="125 kB"
level=info ts=2026-04-28T21:24:57.130508205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="428 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:24:57.14197401Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:24:58.078960639Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:24:58.079467879Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:24:58.097075677Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:24:58.09710257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:24:58.097110925Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:24:58.097150468Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:24:58.100122341Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-28T21:24:58.100142819Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=2
level=debug ts=2026-04-28T21:25:01.432487483Z caller=checkpoint.go:372 msg="writing series" size="822 kB"
level=warn ts=2026-04-28T21:25:03.097584941Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:04.76536644Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:25:08.097688974Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:08.09922343Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:25:08.09924032Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=debug ts=2026-04-28T21:25:11.431761115Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:25:12.725010315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:25:13.097946177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:25:14.151008136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46072"
level=debug ts=2026-04-28T21:25:14.765289479Z caller=checkpoint.go:372 msg="writing series" size="281 kB"
level=debug ts=2026-04-28T21:25:16.930341193Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:25:18.098044443Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:18.098138432Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=info ts=2026-04-28T21:25:19.284200336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=2 total_comp="5.9 MB" avg_comp="3.0 MB" total_uncomp="23 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:25:19.293873033Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:25:21.431963045Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-04-28T21:25:21.949473761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:25:21.954845957Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:25:23.09812351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:24.765742266Z caller=checkpoint.go:372 msg="writing series" size="1000 kB"
level=debug ts=2026-04-28T21:25:24.765765806Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=warn ts=2026-04-28T21:25:28.09774492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:25:28.097749543Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:25:28.097774377Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:25:28.097781908Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:25:28.098875386Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-04-28T21:25:31.432078894Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:25:32.214223477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:25:32.225931195Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:25:33.097828813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:33.140174737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:25:34.765289718Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-28T21:25:35.80832097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:25:35.812192117Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:25:35.812211337Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:25:35.812233397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:25:35.815334117Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:25:36.529527013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:25:36.538280061Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:25:36.538303528Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:25:36.538333031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:25:36.546231394Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:25:38.097423144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:38.098605117Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-04-28T21:25:40.237029248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51656"
level=debug ts=2026-04-28T21:25:41.432053553Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T21:25:42.727802761Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:25:43.098124843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:44.765223295Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T21:25:44.871761871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:25:46.38908475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="428 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:25:46.397468803Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:25:48.097228696Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:48.098589919Z caller=checkpoint.go:372 msg="writing series" size="463 kB"
level=debug ts=2026-04-28T21:25:48.098607122Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-28T21:25:51.211745074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:25:51.218902988Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:25:51.431496757Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-04-28T21:25:52.032193519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:25:52.036430384Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:25:53.098071071Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:54.764875768Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:25:58.079073317Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:25:58.079626975Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:25:58.09742715Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:25:58.097448367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:25:58.097458275Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:25:58.097467822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:25:58.098624467Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=debug ts=2026-04-28T21:26:01.432222815Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:26:03.098124063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:04.765609811Z caller=checkpoint.go:372 msg="writing series" size="710 kB"
level=info ts=2026-04-28T21:26:04.816664874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:26:04.826183224Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=warn ts=2026-04-28T21:26:08.097149494Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:08.098553038Z caller=checkpoint.go:372 msg="writing series" size="762 kB"
level=debug ts=2026-04-28T21:26:08.098571394Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=6
level=debug ts=2026-04-28T21:26:10.305921514Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:26:11.432237026Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-28T21:26:12.730431329Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:26:13.097428961Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:26:13.736667837Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:26:13.74174487Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:26:14.154331501Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:46944"
level=debug ts=2026-04-28T21:26:14.765475383Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=info ts=2026-04-28T21:26:17.502402551Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:26:17.511800691Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:26:18.097543989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:18.098803204Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:26:21.050399174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:26:21.063518737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:26:21.432432439Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:26:23.097143352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:23.455756002Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:26:24.765886923Z caller=checkpoint.go:372 msg="writing series" size="503 kB"
level=debug ts=2026-04-28T21:26:24.765912872Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T21:26:28.097997676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:26:28.098021502Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:26:28.098011534Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:26:28.098030493Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:26:28.098246637Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:26:29.161885722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:26:29.166313947Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:26:31.432251645Z caller=checkpoint.go:372 msg="writing series" size="353 kB"
level=info ts=2026-04-28T21:26:32.430756034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="428 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:26:32.439146482Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:26:33.097491316Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:34.765012382Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=debug ts=2026-04-28T21:26:37.072764657Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:26:37.07280638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:26:37.08053438Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:26:38.097126839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:38.098198162Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-04-28T21:26:40.241402619Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:56488"
level=debug ts=2026-04-28T21:26:41.431579251Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-28T21:26:42.732226824Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:26:43.097428702Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:26:44.155377591Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:60388"
level=debug ts=2026-04-28T21:26:44.765561122Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:26:47.732579555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:26:47.741678984Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:26:48.097485764Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:48.098804815Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-04-28T21:26:48.098820749Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=7
level=info ts=2026-04-28T21:26:50.698951485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:26:50.707157263Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:26:51.432311504Z caller=checkpoint.go:372 msg="writing series" size="91 kB"
level=warn ts=2026-04-28T21:26:53.098162684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:26:54.463607854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:26:54.468810719Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:26:54.765756975Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:26:58.078959273Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:26:58.080094306Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:26:58.097448649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:26:58.09746902Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:26:58.097476198Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:26:58.097467645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:26:58.098629024Z caller=checkpoint.go:372 msg="writing series" size="124 kB"
level=debug ts=2026-04-28T21:27:01.432385887Z caller=checkpoint.go:372 msg="writing series" size="110 kB"
level=warn ts=2026-04-28T21:27:03.097257375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:04.765096013Z caller=checkpoint.go:372 msg="writing series" size="65 kB"
level=info ts=2026-04-28T21:27:06.78717253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:27:06.795696719Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:27:07.431484314Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:27:08.097129646Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:08.098274445Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
ts=2026-04-28T21:27:10.243770371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:35314"
level=info ts=2026-04-28T21:27:10.258998216Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:27:10.270058177Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:27:11.444066938Z caller=checkpoint.go:372 msg="writing series" size="11 MB"
level=debug ts=2026-04-28T21:27:11.444094484Z caller=checkpoint.go:384 msg="flushing series" totalSize="11 MB" series=7
ts=2026-04-28T21:27:12.733701515Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:27:13.097842469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:27:14.422406157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:27:14.427964743Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:27:14.765484502Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T21:27:18.097328691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:18.098500449Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-04-28T21:27:20.450690572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:27:20.458290736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:27:21.431767646Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T21:27:22.069698652Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:27:22.071445069Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T21:27:23.097817126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:24.765536443Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-28T21:27:25.310541532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="428 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:27:25.318937439Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:27:28.097774643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:27:28.09778656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:28.097803327Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:27:28.097810849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:27:28.098948052Z caller=checkpoint.go:372 msg="writing series" size="96 kB"
level=info ts=2026-04-28T21:27:31.174647648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:27:31.184326067Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:27:31.432120315Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=warn ts=2026-04-28T21:27:33.098169537Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:34.765703738Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-28T21:27:35.105031785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:27:38.097238564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:38.09837573Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
ts=2026-04-28T21:27:40.245528286Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47384"
level=debug ts=2026-04-28T21:27:41.43225812Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-04-28T21:27:41.687815232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:27:41.696634011Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:27:42.73719507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:27:43.098028278Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:27:44.158739814Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43828"
level=debug ts=2026-04-28T21:27:44.765061276Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=warn ts=2026-04-28T21:27:48.098024968Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:48.099139017Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-28T21:27:51.432551744Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=debug ts=2026-04-28T21:27:51.43257656Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-28T21:27:53.097956332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:54.765113996Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:27:58.079349968Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:27:58.080288743Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:27:58.098002378Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:27:58.098027153Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:27:58.09803838Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:27:58.098051671Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:27:58.099759163Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T21:27:58.099771654Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=2
level=info ts=2026-04-28T21:28:00.488968879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:28:00.497606714Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:28:00.497654663Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:28:00.497677898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:28:00.50532417Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:28:01.443876224Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-28T21:28:01.44390205Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=1
level=warn ts=2026-04-28T21:28:03.097310417Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:28:03.407913288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:28:03.412122419Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:28:03.70665664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="9.0 MB" avg_comp="2.3 MB" total_uncomp="78 MB" avg_uncomp="20 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:28:03.7137804Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:28:04.764979184Z caller=checkpoint.go:372 msg="writing series" size="220 kB"
level=info ts=2026-04-28T21:28:04.766923937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:28:04.771609982Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:28:07.169467285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:28:07.215564329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=18 total_comp="52 MB" avg_comp="2.9 MB" total_uncomp="428 MB" avg_uncomp="24 MB" full=18 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:28:07.223236332Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 18, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:28:08.097364736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:28:08.098456647Z caller=checkpoint.go:372 msg="writing series" size="643 B"
ts=2026-04-28T21:28:10.248320302Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51116"
level=debug ts=2026-04-28T21:28:11.431908863Z caller=checkpoint.go:372 msg="writing series" size="334 kB"
ts=2026-04-28T21:28:12.739928407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:28:13.097509971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:28:14.204029011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:28:14.765656978Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:28:16.457072147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:28:16.464526088Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:28:18.097699129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:28:18.099222872Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:28:18.099239653Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-04-28T21:28:21.432390563Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=warn ts=2026-04-28T21:28:23.09769039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:28:24.765493377Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=info ts=2026-04-28T21:28:27.054671139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:28:27.065854274Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=info ts=2026-04-28T21:28:28.097743682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:28:28.097744458Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:28:28.097770358Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:28:28.097778547Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:28:28.09889352Z caller=checkpoint.go:384 msg="flushing series" totalSize="283 kB" series=2
level=info ts=2026-04-28T21:28:28.103471948Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-28T21:28:28.105135272Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.007095301s
level=info ts=2026-04-28T21:28:28.826831412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:28:28.83197647Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:28:33.097893577Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:28:38.097213655Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:28:40.250516538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:39808"
level=debug ts=2026-04-28T21:28:40.939279163Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:28:42.741884081Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:28:43.097571673Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:28:43.57780315Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:28:43.590138259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:28:48.097268023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:28:50.087067557Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:28:50.095844377Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:28:53.097344663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:28:58.079087301Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:28:58.079109647Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T21:28:58.097441464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:28:58.097452549Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:28:58.097472339Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:28:58.09747927Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:28:58.097472286Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:28:58.097593705Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-28T21:28:58.097604634Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:28:58.09764384Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=33.633µs success=true
level=debug ts=2026-04-28T21:28:58.097865685Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-28T21:28:59.730954247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:29:00.221746192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:29:00.227916167Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:29:01.432270613Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T21:29:03.097209143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:29:03.662099347Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:29:03.667512041Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:29:04.404429021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:29:04.411340249Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:29:04.411366615Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:29:04.411406357Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:29:04.417874853Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:29:04.823227067Z caller=checkpoint.go:372 msg="writing series" size="56 MB"
level=debug ts=2026-04-28T21:29:04.823253815Z caller=checkpoint.go:384 msg="flushing series" totalSize="56 MB" series=3
level=warn ts=2026-04-28T21:29:08.098209978Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:08.098392868Z caller=checkpoint.go:372 msg="writing series" size="521 kB"
level=debug ts=2026-04-28T21:29:11.433088779Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-28T21:29:11.433111699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
ts=2026-04-28T21:29:12.744054905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:29:13.097269996Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:14.765416032Z caller=checkpoint.go:372 msg="writing series" size="260 kB"
level=debug ts=2026-04-28T21:29:14.940024207Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:29:18.097446048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:18.098597035Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-04-28T21:29:20.209929363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:29:20.219942747Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=debug ts=2026-04-28T21:29:21.431968437Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T21:29:21.693919957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:29:21.701850643Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:29:23.09746589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:24.76884984Z caller=checkpoint.go:372 msg="writing series" size="3.8 MB"
level=debug ts=2026-04-28T21:29:24.768872561Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.2 MB" series=4
level=info ts=2026-04-28T21:29:28.097246477Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:29:28.097273747Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:29:28.097281596Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:29:28.097294777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:28.098738753Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:29:28.442592075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:29:28.447354509Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:29:31.432127216Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:29:33.035766328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:29:33.044041248Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:29:33.098010498Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:34.765947091Z caller=checkpoint.go:372 msg="writing series" size="750 kB"
level=debug ts=2026-04-28T21:29:34.765968275Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-28T21:29:38.097969162Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:38.098059985Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:29:39.700519882Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:29:41.432355765Z caller=checkpoint.go:372 msg="writing series" size="108 kB"
ts=2026-04-28T21:29:42.7459859Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:29:43.09811049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:44.765390941Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-04-28T21:29:45.017185775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:29:45.028316014Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:29:45.679234308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59706"
level=info ts=2026-04-28T21:29:46.405856135Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:29:46.411921472Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:29:48.09795414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:48.098012125Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T21:29:49.124259748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:29:49.132837264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:29:51.434437476Z caller=checkpoint.go:372 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-28T21:29:51.434458885Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=5
level=warn ts=2026-04-28T21:29:53.097461227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:54.618568655Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:29:54.765399083Z caller=checkpoint.go:372 msg="writing series" size="186 kB"
level=info ts=2026-04-28T21:29:55.789548414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:29:55.794112472Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:29:58.07895889Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:29:58.079418583Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:29:58.097451041Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:29:58.097475913Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:29:58.097485302Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:29:58.097557493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:29:58.102503982Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-28T21:29:58.102517996Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=debug ts=2026-04-28T21:30:01.43279465Z caller=checkpoint.go:372 msg="writing series" size="905 kB"
level=warn ts=2026-04-28T21:30:03.097688648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:04.764651839Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:30:08.09820727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:08.098727193Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:30:08.098745946Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-04-28T21:30:11.432475686Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-04-28T21:30:12.432496044Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:30:12.74770663Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:30:13.097450532Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:14.765201349Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=warn ts=2026-04-28T21:30:18.097645227Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:18.098732673Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=info ts=2026-04-28T21:30:18.185043194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:30:18.192188813Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:30:18.192211536Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:30:18.192239132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:30:18.196263651Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:30:19.236272159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=d12534e34f1f23d9 immediate=false num_chunks=3 total_comp="8.9 MB" avg_comp="3.0 MB" total_uncomp="35 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:30:19.24445865Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d12534e34f1f23d9/19dd5e0f8ad%3A19dd5edad44%3A52a7d907?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-dqbh\"}"
level=error ts=2026-04-28T21:30:19.244482874Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:30:19.244507021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:30:19.251137827Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:30:21.43199072Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-04-28T21:30:23.097927237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:24.765419616Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:30:24.765441123Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-04-28T21:30:28.097978577Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:30:28.098006533Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:30:28.098014112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:30:28.098015558Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-04-28T21:30:28.098036771Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:30:29.219717999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:30:29.228601913Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:30:29.334975638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:30:29.343123018Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:30:31.432190165Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:30:32.383246917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:30:32.391366937Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:30:33.097390355Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:34.530324312Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:30:34.764974204Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=info ts=2026-04-28T21:30:36.57075933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:30:36.575061083Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:30:36.920995362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:30:36.927314713Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T21:30:38.097404627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:38.098580635Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:30:40.561847388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:30:40.569130534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:30:40.712703146Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:30:41.431742394Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T21:30:42.749863998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:30:43.097880455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:44.765105949Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
ts=2026-04-28T21:30:45.682594806Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39200"
level=warn ts=2026-04-28T21:30:48.097261868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:48.098684638Z caller=checkpoint.go:372 msg="writing series" size="376 kB"
level=debug ts=2026-04-28T21:30:48.098698252Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-28T21:30:51.43210812Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T21:30:53.097935359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:30:54.579761821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:30:54.587156142Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:30:54.764786121Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:30:58.079559235Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:30:58.080032944Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-04-28T21:30:58.080245431Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T21:30:58.080287727Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974901 n=1
level=debug ts=2026-04-28T21:30:58.080334856Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974901 n=1
level=info ts=2026-04-28T21:30:58.098150246Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:30:58.098151312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:30:58.0981724Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:30:58.098180555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:30:58.098210083Z caller=checkpoint.go:372 msg="writing series" size="130 kB"
level=debug ts=2026-04-28T21:31:01.431425359Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:31:03.097689539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:04.764976781Z caller=checkpoint.go:372 msg="writing series" size="481 kB"
level=debug ts=2026-04-28T21:31:07.425433583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:31:08.097936015Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:08.098423864Z caller=checkpoint.go:372 msg="writing series" size="762 kB"
level=debug ts=2026-04-28T21:31:08.098438588Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
ts=2026-04-28T21:31:10.257405628Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59248"
level=info ts=2026-04-28T21:31:10.622223646Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:31:10.632302487Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:31:11.431906977Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
ts=2026-04-28T21:31:12.752429758Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T21:31:12.957749776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:31:12.962596977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:31:13.098096904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:31:14.175016445Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34114"
level=info ts=2026-04-28T21:31:14.697988114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:31:14.703746353Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:31:14.764864046Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
ts=2026-04-28T21:31:15.684560384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60772"
level=debug ts=2026-04-28T21:31:16.190346152Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:31:16.192053175Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T21:31:18.098049211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:18.098369047Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:31:18.727021351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:31:18.734477751Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:31:19.932014215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:31:19.939280791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:31:21.432357582Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:31:23.097175281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:24.764835109Z caller=checkpoint.go:372 msg="writing series" size="460 kB"
level=debug ts=2026-04-28T21:31:24.76485671Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T21:31:28.097351913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:31:28.097374783Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:31:28.097382446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:31:28.097380761Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:28.098573127Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:31:30.715066066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:31:30.724786293Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:31:31.432070215Z caller=checkpoint.go:372 msg="writing series" size="452 kB"
level=warn ts=2026-04-28T21:31:33.097309742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:34.765205385Z caller=checkpoint.go:372 msg="writing series" size="261 kB"
level=debug ts=2026-04-28T21:31:36.996512823Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:31:38.098138062Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:38.098146485Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
ts=2026-04-28T21:31:40.25937406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47208"
level=debug ts=2026-04-28T21:31:41.432406304Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:31:41.43242571Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
ts=2026-04-28T21:31:42.755023204Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:31:43.097747898Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:31:44.176048572Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37408"
level=debug ts=2026-04-28T21:31:44.765038332Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:31:45.178522321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:31:45.185789651Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:31:48.097367144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:48.098703985Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-04-28T21:31:51.432085043Z caller=checkpoint.go:372 msg="writing series" size="94 kB"
level=warn ts=2026-04-28T21:31:53.09730971Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:54.7650568Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=debug ts=2026-04-28T21:31:58.078889862Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:31:58.079973601Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T21:31:58.09709823Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:31:58.09712296Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:31:58.097131528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:31:58.09713497Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:31:58.09830687Z caller=checkpoint.go:372 msg="writing series" size="131 kB"
level=debug ts=2026-04-28T21:31:59.375725783Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:31:59.461739174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:31:59.470731568Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:32:01.432032554Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=info ts=2026-04-28T21:32:01.498416187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:32:01.502512039Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:32:02.090499426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:32:02.095955399Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:32:03.097496895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:04.76508885Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=info ts=2026-04-28T21:32:07.978865529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:32:07.987845811Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:32:08.098014505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:08.098086506Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=debug ts=2026-04-28T21:32:11.444641442Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-04-28T21:32:11.444665689Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=9
ts=2026-04-28T21:32:12.756488909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=info ts=2026-04-28T21:32:12.845361089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:32:12.859308024Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:32:12.901748306Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:32:13.098214572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:14.765276577Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T21:32:18.097926259Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:18.09802039Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-04-28T21:32:21.431683621Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:32:23.097191005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:24.765104391Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-28T21:32:28.09809092Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:32:28.09811799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:32:28.098126331Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:32:28.098159005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:28.098209425Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-04-28T21:32:29.495667864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:32:29.504427143Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:32:29.504451505Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:32:29.504475284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:32:29.511425286Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:32:31.431431699Z caller=checkpoint.go:372 msg="writing series" size="49 kB"
level=warn ts=2026-04-28T21:32:33.097896246Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:34.765267168Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:32:36.546991782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:32:36.55140586Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:32:36.55142865Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:32:36.551453449Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:32:36.555135877Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:32:38.09717727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:38.098315451Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T21:32:41.43207134Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-04-28T21:32:42.757440466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:32:43.098155513Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:44.765013258Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-04-28T21:32:44.76814016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:32:44.776337824Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:32:44.776358134Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:32:44.77638138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:32:44.781144215Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:32:45.833989052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:32:45.841154226Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:32:45.930648817Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:32:48.098098793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:48.098149994Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=debug ts=2026-04-28T21:32:49.473772409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:32:50.558739522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:32:50.565143307Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:32:51.432362956Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=info ts=2026-04-28T21:32:51.603813455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:32:51.608455911Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:32:53.098103457Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:54.764601569Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-04-28T21:32:55.928053392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:32:55.933504734Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:32:57.282168106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:32:57.289534344Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:32:58.079091438Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:32:58.079161024Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:32:58.097879119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:32:58.09788069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:32:58.097903065Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:32:58.097913866Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:32:58.099837719Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-28T21:32:58.099850276Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.6 MB" series=14
level=info ts=2026-04-28T21:32:59.193271003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:32:59.199920342Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:33:01.436084491Z caller=checkpoint.go:372 msg="writing series" size="13 MB"
level=debug ts=2026-04-28T21:33:01.436106828Z caller=checkpoint.go:384 msg="flushing series" totalSize="13 MB" series=1
level=warn ts=2026-04-28T21:33:03.097920622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:04.040203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:33:04.049742548Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:33:04.765201836Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=warn ts=2026-04-28T21:33:08.097504196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:33:08.098570009Z caller=checkpoint.go:372 msg="writing series" size="643 B"
level=info ts=2026-04-28T21:33:08.61684167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:33:08.62395416Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:33:11.432428586Z caller=checkpoint.go:372 msg="writing series" size="326 kB"
ts=2026-04-28T21:33:12.759147914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:33:13.097882396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:33:14.765564597Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:33:15.697963119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60542"
level=debug ts=2026-04-28T21:33:16.478242612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:33:18.097960735Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:33:18.098647313Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:33:18.098662921Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-04-28T21:33:21.43179896Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=warn ts=2026-04-28T21:33:23.097992523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:23.637591042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:33:23.646761395Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:33:24.765322816Z caller=checkpoint.go:372 msg="writing series" size="122 kB"
level=info ts=2026-04-28T21:33:28.097070556Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:33:28.097093632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:33:28.097101821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:33:28.097127983Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:33:28.098125442Z caller=checkpoint.go:384 msg="flushing series" totalSize="319 kB" series=2
level=info ts=2026-04-28T21:33:28.105039807Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-28T21:33:28.106746287Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008945243s
level=warn ts=2026-04-28T21:33:33.097665563Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:33.756914524Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:33:33.762204146Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:33:37.124463912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:33:38.097368107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:41.957244203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:33:41.961775329Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:33:42.761025247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:33:43.097352656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:47.557043205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:33:47.566110873Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:33:48.097165466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:49.341805005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:33:49.34185753Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-28T21:33:49.346018105Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:33:49.748208269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:33:49.754541855Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:33:51.842768782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:33:51.85000531Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:33:53.097324379Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:58.079251315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:33:58.079693875Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:33:58.097152614Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:33:58.097168829Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:33:58.09717622Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:33:58.097170701Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-28T21:33:58.0971992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:33:58.097303427Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-28T21:33:58.097311378Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:33:58.097329196Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=13.596µs success=true
level=debug ts=2026-04-28T21:33:58.09751785Z caller=checkpoint.go:372 msg="writing series" size="3.7 kB"
level=debug ts=2026-04-28T21:34:01.431195331Z caller=checkpoint.go:372 msg="writing series" size="6.9 kB"
level=info ts=2026-04-28T21:34:02.21033353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:34:02.219061393Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:34:03.097315813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:04.817985788Z caller=checkpoint.go:372 msg="writing series" size="58 MB"
level=debug ts=2026-04-28T21:34:04.818025652Z caller=checkpoint.go:384 msg="flushing series" totalSize="58 MB" series=3
level=info ts=2026-04-28T21:34:06.891745983Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:34:06.897806651Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:34:08.097464004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:08.098101542Z caller=checkpoint.go:372 msg="writing series" size="510 kB"
level=debug ts=2026-04-28T21:34:11.433067009Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-28T21:34:11.433087584Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=2
ts=2026-04-28T21:34:12.763548892Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:34:13.097686211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:14.765216944Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=debug ts=2026-04-28T21:34:14.901331316Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:34:16.955899498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:34:16.96278483Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:34:18.097589331Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:18.09766269Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-04-28T21:34:20.612767574Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:34:20.612838648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="879 kB" avg_comp="879 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:34:20.61788475Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:34:21.431244255Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:34:23.098224899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:34:23.85459053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:34:23.858946585Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:34:24.766948792Z caller=checkpoint.go:372 msg="writing series" size="4.0 MB"
level=debug ts=2026-04-28T21:34:24.76697009Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=4
level=info ts=2026-04-28T21:34:28.097880429Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:34:28.097907885Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:34:28.097915843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:34:28.097963134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:28.098122322Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:34:30.916264743Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:34:30.920798925Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:34:31.431186846Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:34:32.384242608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=19 total_comp="55 MB" avg_comp="2.9 MB" total_uncomp="452 MB" avg_uncomp="24 MB" full=19 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:34:32.39269147Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 19, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:34:33.097927782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:34.765401142Z caller=checkpoint.go:372 msg="writing series" size="889 kB"
level=debug ts=2026-04-28T21:34:34.765424676Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=3
level=info ts=2026-04-28T21:34:35.514867649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:34:35.520526928Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:34:37.475952066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:34:37.481761708Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:34:38.097975827Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:38.097978217Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
ts=2026-04-28T21:34:40.272678859Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:45898"
level=debug ts=2026-04-28T21:34:41.431817312Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
ts=2026-04-28T21:34:42.765531522Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:34:43.097645763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:34:44.272518084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:34:44.28028447Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:34:44.764712992Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-04-28T21:34:46.579014981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:34:46.587227555Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:34:46.587251696Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:34:46.587276004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:34:46.594881722Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:34:48.097473865Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:48.097544668Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=debug ts=2026-04-28T21:34:51.433005507Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-28T21:34:51.433025913Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.8 MB" series=5
level=warn ts=2026-04-28T21:34:53.097495003Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:53.753740619Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:34:54.764687224Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:34:56.200600026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:34:56.208998448Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:34:58.078974082Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:34:58.079386961Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:34:58.098062215Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:34:58.098068111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:34:58.09808266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:34:58.098093915Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:34:58.100831744Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-28T21:34:58.100843795Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=2
level=debug ts=2026-04-28T21:35:01.432142167Z caller=checkpoint.go:372 msg="writing series" size="802 kB"
level=info ts=2026-04-28T21:35:01.771389111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:35:01.777181423Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T21:35:03.098179212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:35:03.504846442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:35:03.515499585Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:35:04.765270861Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:35:05.849480049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:35:05.856777612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:35:08.097677509Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:08.098001867Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:35:08.09801613Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
ts=2026-04-28T21:35:10.275003358Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47544"
level=debug ts=2026-04-28T21:35:11.431829441Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:35:12.768012153Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:35:13.097281927Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:35:13.510563311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:35:13.515123348Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:35:14.218865987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:35:14.220721126Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:35:14.765148938Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=info ts=2026-04-28T21:35:17.173730539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:35:17.1777942Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:35:18.097855914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:18.097894643Z caller=checkpoint.go:372 msg="writing series" size="6.1 kB"
level=debug ts=2026-04-28T21:35:21.431316101Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T21:35:23.097799126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:24.765373958Z caller=checkpoint.go:372 msg="writing series" size="961 kB"
level=debug ts=2026-04-28T21:35:24.765400509Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=debug ts=2026-04-28T21:35:24.8031269Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:35:27.917453388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:35:27.928810651Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:35:27.92883338Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:35:27.928871529Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:35:27.937165836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:35:28.097488061Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:35:28.09751223Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:35:28.097520062Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:35:28.097511941Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=warn ts=2026-04-28T21:35:28.097521279Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:35:29.427865491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:35:29.435791294Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:35:29.645398629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:35:29.649983063Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:35:31.431648848Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-04-28T21:35:32.866227384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:35:32.875287323Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:35:33.097643795Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:34.764695537Z caller=checkpoint.go:372 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T21:35:38.098005367Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:38.098150948Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-04-28T21:35:40.276846807Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40636"
level=debug ts=2026-04-28T21:35:41.43124208Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T21:35:42.770220974Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:35:43.097369455Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:44.765062963Z caller=checkpoint.go:372 msg="writing series" size="12 kB"
level=info ts=2026-04-28T21:35:45.766034412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:35:45.778047013Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:35:48.097136403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:48.098676985Z caller=checkpoint.go:372 msg="writing series" size="484 kB"
level=debug ts=2026-04-28T21:35:48.098692373Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-28T21:35:49.086543973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:35:49.094117684Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:35:51.06156524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:35:51.431135516Z caller=checkpoint.go:372 msg="writing series" size="66 kB"
level=debug ts=2026-04-28T21:35:52.417124125Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:35:53.097320446Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:35:54.476954541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:35:54.481602486Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:35:54.765222271Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:35:57.469768727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:35:57.475001916Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:35:58.079002038Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:35:58.079399461Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:35:58.09795982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:35:58.097977503Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:35:58.09798528Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:35:58.098000267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:35:58.098040583Z caller=checkpoint.go:372 msg="writing series" size="133 kB"
level=debug ts=2026-04-28T21:36:01.431318888Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=warn ts=2026-04-28T21:36:03.098072353Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:36:03.282060638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:36:03.286483369Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:36:03.819522421Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:36:03.823908916Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:36:04.765331914Z caller=checkpoint.go:372 msg="writing series" size="658 kB"
level=warn ts=2026-04-28T21:36:08.097392899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:08.099914233Z caller=checkpoint.go:372 msg="writing series" size="620 kB"
level=debug ts=2026-04-28T21:36:08.099948042Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=6
level=info ts=2026-04-28T21:36:11.426954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:36:11.431312993Z caller=checkpoint.go:372 msg="writing series" size="5.3 kB"
level=error ts=2026-04-28T21:36:11.433057424Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:36:12.28521674Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:36:12.289817387Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T21:36:12.772410754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:36:13.0979449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:36:14.200691306Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:52878"
level=debug ts=2026-04-28T21:36:14.764867389Z caller=checkpoint.go:372 msg="writing series" size="2.4 kB"
level=info ts=2026-04-28T21:36:16.070887242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:36:16.080823438Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:36:18.09779283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:18.097981259Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:36:18.268179831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:36:18.276375792Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:36:21.431372103Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-28T21:36:22.363444961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:36:23.09729215Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:24.765222068Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=debug ts=2026-04-28T21:36:24.765245366Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T21:36:28.09796496Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:36:28.097993121Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:36:28.098002097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:36:28.098005262Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:28.098017262Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:36:29.122321894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:36:29.132769559Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:36:31.431922776Z caller=checkpoint.go:372 msg="writing series" size="364 kB"
level=warn ts=2026-04-28T21:36:33.097177529Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:34.764528488Z caller=checkpoint.go:372 msg="writing series" size="350 kB"
level=debug ts=2026-04-28T21:36:38.097728466Z caller=checkpoint.go:372 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T21:36:38.097806244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:41.397383585Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:36:41.431081044Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:36:41.431101153Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T21:36:41.629886125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:36:41.636944476Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-04-28T21:36:42.774364574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:36:43.09727729Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:44.764482589Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-28T21:36:48.097728548Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:36:48.097934654Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T21:36:48.11688694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:36:48.121150056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:36:48.121167184Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:36:48.121189626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:36:48.124503871Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:36:50.279686539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:36:50.300976226Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:36:51.431089958Z caller=checkpoint.go:372 msg="writing series" size="97 kB"
level=debug ts=2026-04-28T21:36:51.571737669Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:36:53.097607421Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:36:53.246895251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:36:53.252153657Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:36:54.764257054Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:36:55.811840287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:36:55.817971851Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:36:55.817993244Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:36:55.818021016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:36:55.823382784Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:36:58.078958773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:36:58.0797572Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T21:36:58.097801302Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:36:58.097823525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:36:58.097842936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:36:58.097850608Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:36:58.09796264Z caller=checkpoint.go:372 msg="writing series" size="136 kB"
level=info ts=2026-04-28T21:36:58.908665949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:36:58.916141289Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:37:01.431913569Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=info ts=2026-04-28T21:37:02.594218081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:37:02.598325411Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:37:03.097686201Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:37:03.919038559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:37:03.923318343Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:37:04.764833744Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T21:37:08.097860471Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:08.097875279Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=debug ts=2026-04-28T21:37:08.097891427Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-28T21:37:08.321483835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:37:08.328288925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:37:11.447953945Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:37:11.447976657Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-04-28T21:37:12.776246817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:37:13.097675843Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:14.764877486Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-04-28T21:37:16.188328054Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:37:16.7979906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:37:16.811530165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:37:18.098032535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:18.098138823Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=debug ts=2026-04-28T21:37:21.431080514Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T21:37:22.395531936Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:37:22.406770615Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:37:23.097352784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:24.764847861Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:37:27.00851224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:37:27.014092881Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:37:28.09717118Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:37:28.097195852Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:37:28.097203409Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:37:28.097206845Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:28.098483331Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=debug ts=2026-04-28T21:37:31.431741479Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=warn ts=2026-04-28T21:37:33.097774407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:34.765262851Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-28T21:37:38.097513689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:38.097523454Z caller=checkpoint.go:372 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T21:37:40.757398435Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:37:40.970222038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:37:40.978936177Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:37:41.425528713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:37:41.429851356Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:37:41.430890939Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=debug ts=2026-04-28T21:37:42.122115678Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:37:42.665266293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:37:42.669851012Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T21:37:42.778905259Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:37:43.097151217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:44.764959672Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-04-28T21:37:45.714733047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:48940"
level=warn ts=2026-04-28T21:37:48.098060158Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:48.098068517Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-28T21:37:48.88420509Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:37:48.888133206Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:37:50.140228394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:37:50.145149093Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:37:50.735967439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:37:50.743554359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:37:51.43189823Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
level=warn ts=2026-04-28T21:37:53.098078745Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:54.765167387Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=debug ts=2026-04-28T21:37:54.765187907Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-04-28T21:37:58.079212178Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:37:58.079646227Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:37:58.098024128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:37:58.098046276Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:37:58.098054448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:37:58.09805757Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:37:58.099086384Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-28T21:37:58.099101183Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=1
level=debug ts=2026-04-28T21:38:01.444563004Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:38:01.444602025Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=warn ts=2026-04-28T21:38:03.097865449Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:38:04.765135946Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-04-28T21:38:06.546429091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:38:06.555918395Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:38:08.097608981Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:38:08.097669983Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=debug ts=2026-04-28T21:38:08.913372742Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:38:10.289374367Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:50964"
level=info ts=2026-04-28T21:38:11.399681875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:38:11.408327817Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:38:11.431449665Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-04-28T21:38:12.781546278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:38:13.097389847Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:38:14.208402756Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57058"
level=debug ts=2026-04-28T21:38:14.764634146Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-28T21:38:18.097436752Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:38:18.099356516Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:38:18.099371154Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-04-28T21:38:21.431590379Z caller=checkpoint.go:372 msg="writing series" size="207 kB"
level=warn ts=2026-04-28T21:38:23.097863648Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:38:24.764734552Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-04-28T21:38:26.603193592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:38:26.611857523Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:38:28.097520474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:38:28.097545539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:38:28.097552641Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:38:28.097543904Z caller=checkpoint.go:384 msg="flushing series" totalSize="344 kB" series=2
level=warn ts=2026-04-28T21:38:28.097597338Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:38:28.101374063Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000010.tmp new=/var/loki/wal/checkpoint.000010
level=info ts=2026-04-28T21:38:28.103041313Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.005584507s
level=info ts=2026-04-28T21:38:29.687117194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=1 total_comp="2.2 MB" avg_comp="2.2 MB" total_uncomp="14 MB" avg_uncomp="14 MB" synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:38:29.696075656Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:38:30.976824924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:38:30.981315734Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:38:31.40846363Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:38:31.412583558Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:38:33.09809274Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:38:37.138725963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:38:38.097717892Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:38:40.546678588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:38:40.551011245Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:38:41.361398138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:38:42.784128676Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:38:43.097384538Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:38:45.33724403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:38:45.343078161Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:38:48.009879686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:38:48.017187959Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:38:48.097912935Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:38:53.09781519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:38:57.109118758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:38:57.11910063Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:38:58.079214292Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:38:58.079581481Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:38:58.097473749Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:38:58.097494745Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:38:58.097502847Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:38:58.097501469Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:38:58.097511741Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:38:58.097658429Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-28T21:38:58.097665831Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:38:58.097686441Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.496µs success=true
level=debug ts=2026-04-28T21:38:58.097918269Z caller=checkpoint.go:372 msg="writing series" size="4.0 kB"
level=debug ts=2026-04-28T21:39:01.432238392Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:39:03.098219314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:04.824297244Z caller=checkpoint.go:372 msg="writing series" size="59 MB"
level=debug ts=2026-04-28T21:39:04.824326633Z caller=checkpoint.go:384 msg="flushing series" totalSize="59 MB" series=3
level=info ts=2026-04-28T21:39:05.945007071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:39:05.954029986Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:39:05.954053358Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:39:05.954079174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:39:05.961925598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:39:08.097957304Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:08.098701744Z caller=checkpoint.go:372 msg="writing series" size="678 kB"
ts=2026-04-28T21:39:10.294808717Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60894"
level=debug ts=2026-04-28T21:39:10.371564605Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:39:11.435022283Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-28T21:39:11.435055418Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=2
ts=2026-04-28T21:39:12.786576118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=info ts=2026-04-28T21:39:13.089391924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:39:13.096769352Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T21:39:13.097774609Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:14.765866034Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=warn ts=2026-04-28T21:39:18.098143705Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:18.098202611Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-04-28T21:39:20.173845433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:39:20.180518026Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:39:21.173663434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:39:21.178478256Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:39:21.431647259Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:39:23.098123234Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:39:23.875862628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:39:23.884249536Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:39:24.769505036Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-04-28T21:39:24.769530709Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=4
level=info ts=2026-04-28T21:39:26.164568007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:39:26.168853733Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:39:27.550158117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:39:27.556862311Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:39:27.55688664Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:39:27.556911202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:39:27.560806824Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:39:28.097559037Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:39:28.097566172Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:28.097594643Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:39:28.097603188Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:39:28.0989312Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:39:30.100102996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:39:30.108581542Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:39:30.158589631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:39:30.163781563Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:39:30.163801096Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:39:30.163824612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:39:30.169638962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:39:31.43215685Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:39:33.097180736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:34.766397814Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-28T21:39:34.766421124Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=3
level=debug ts=2026-04-28T21:39:35.219921867Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:39:35.219978581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="112 B" avg_comp="112 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:39:35.225439247Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:39:38.097374645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:38.098524119Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:39:41.432147702Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-04-28T21:39:42.066314701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:39:42.071699333Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
ts=2026-04-28T21:39:42.788480431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T21:39:42.852793697Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:39:42.854411801Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T21:39:43.097608138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:39:43.447372678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:39:43.451662215Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:39:44.764991352Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
ts=2026-04-28T21:39:45.722235521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:32904"
level=info ts=2026-04-28T21:39:45.874366504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:39:45.882227098Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:39:48.098147226Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:48.098170642Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T21:39:48.251511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:39:48.259797589Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:39:51.433313001Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-28T21:39:51.43333565Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.9 MB" series=5
level=info ts=2026-04-28T21:39:52.127626126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:39:52.131131944Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:39:53.09796759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:54.764967009Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:39:58.07896745Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:39:58.079290095Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:39:58.09765386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:39:58.097665005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:39:58.097678355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:39:58.09768868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:39:58.103283935Z caller=checkpoint.go:372 msg="writing series" size="5.9 MB"
level=debug ts=2026-04-28T21:39:58.103303215Z caller=checkpoint.go:384 msg="flushing series" totalSize="6.1 MB" series=2
level=info ts=2026-04-28T21:39:59.132336784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:39:59.141453891Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:40:01.43207035Z caller=checkpoint.go:372 msg="writing series" size="906 kB"
level=warn ts=2026-04-28T21:40:03.097127972Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:04.764714511Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:40:08.097857217Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:08.098259966Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:40:08.098274499Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-04-28T21:40:08.566595072Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:40:08.56660314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="203 B" avg_comp="203 B" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" idle=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:40:08.5732933Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:40:08.867140718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="260 B" avg_comp="260 B" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" idle=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:40:08.870921748Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:40:09.16684481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="165 B" avg_comp="165 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" idle=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:40:09.170556689Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:40:09.467240193Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="153 B" avg_comp="153 B" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" idle=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:40:09.4710155Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:40:09.766918481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="192 B" avg_comp="192 B" total_uncomp="11 kB" avg_uncomp="11 kB" idle=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:40:09.770549035Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:40:10.339223386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:40:10.343674125Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:40:10.696320358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:40:10.700890351Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:40:11.432042439Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:40:12.791115168Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:40:13.098080591Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:14.764925466Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=warn ts=2026-04-28T21:40:18.097970435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:18.097988544Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-04-28T21:40:18.116107931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:40:18.120879973Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:40:19.279321877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:40:19.284289933Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:40:20.111766052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:40:20.115274536Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:40:21.432433Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-04-28T21:40:21.443353776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:40:21.446917457Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:40:21.775092604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:40:21.778625656Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:40:23.097313524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:24.765759604Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-28T21:40:24.765792665Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-04-28T21:40:25.190778867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:40:25.195307727Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T21:40:25.195328476Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:40:25.195352781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:40:25.199259598Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:40:25.529211511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:40:25.533348314Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:40:25.744903285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:40:25.74833163Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:40:26.697724582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:40:26.704732594Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:40:27.147010539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:40:27.150287176Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:40:28.097198034Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:40:28.097219929Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:40:28.09722769Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:40:28.097227493Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:28.098433445Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=debug ts=2026-04-28T21:40:31.432175784Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T21:40:33.097728069Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:34.764880118Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-28T21:40:38.097803391Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:38.09906613Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:40:38.265245731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:40:38.273538457Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:40:40.45034497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:40:41.432014095Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T21:40:42.792800275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:40:43.097696511Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:40:43.471834492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:40:43.475133705Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:40:44.666120044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:40:44.765520066Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T21:40:45.752989098Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:40:45.760415391Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:40:46.533577252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:40:46.541675523Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:40:48.097396402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:48.098858038Z caller=checkpoint.go:372 msg="writing series" size="427 kB"
level=debug ts=2026-04-28T21:40:48.09887285Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-04-28T21:40:49.344029335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:40:49.352170601Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:40:51.431342458Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=info ts=2026-04-28T21:40:52.091561099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:40:52.095984308Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:40:52.681868227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:40:52.685678105Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T21:40:53.097942095Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:54.765048566Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:40:58.079750341Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T21:40:58.080020681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:40:58.080237651Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=error ts=2026-04-28T21:40:58.08406434Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:40:58.09771936Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:40:58.097737529Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:40:58.097747121Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:40:58.097787963Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:40:58.098896043Z caller=checkpoint.go:372 msg="writing series" size="134 kB"
level=info ts=2026-04-28T21:40:58.192380882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:40:58.196466778Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:40:59.179848541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:40:59.18369742Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T21:41:01.432302912Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T21:41:01.679224121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:41:01.68606168Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:41:02.316817361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:41:02.320424164Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T21:41:03.098133778Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:04.765275563Z caller=checkpoint.go:372 msg="writing series" size="462 kB"
level=info ts=2026-04-28T21:41:07.119427116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:41:07.123704642Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:41:07.986435022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:41:07.990022896Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:41:08.097476124Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:08.098921723Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-04-28T21:41:08.098937061Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
ts=2026-04-28T21:41:10.302300673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55316"
level=debug ts=2026-04-28T21:41:11.432010097Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-04-28T21:41:11.910918796Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:41:12.794843879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:41:13.098046502Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:41:14.147836278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:41:14.154466448Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-04-28T21:41:14.154485831Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:41:14.154509679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:41:14.15853228Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:41:14.765197824Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
ts=2026-04-28T21:41:15.728187672Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49050"
level=warn ts=2026-04-28T21:41:18.098003281Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:18.098183838Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:41:19.921931011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:41:19.930011773Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:41:19.930033126Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:41:19.930059248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:41:19.936959929Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:41:21.432244438Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:41:23.097241416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:24.765948234Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=debug ts=2026-04-28T21:41:24.76597013Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-04-28T21:41:26.106504794Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:41:28.097775886Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:41:28.097799898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:41:28.097815975Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:41:28.097835266Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:28.098061575Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:41:30.50339515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:41:30.51256585Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:41:31.218561004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:41:31.223484442Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:41:31.432330405Z caller=checkpoint.go:372 msg="writing series" size="501 kB"
level=info ts=2026-04-28T21:41:32.357490168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:41:32.365837813Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:41:32.813189122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:41:32.817975436Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:41:33.097754104Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:34.764952302Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-04-28T21:41:36.953574077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:41:36.960685204Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:41:37.767070798Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:41:37.774542069Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:41:38.09785965Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:38.097900078Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-04-28T21:41:38.810382412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:41:38.814719845Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-28T21:41:40.304295285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:42536"
level=info ts=2026-04-28T21:41:41.130132951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:41:41.133593594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T21:41:41.432348319Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:41:41.432371376Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-28T21:41:41.719327043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:41:41.72283082Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:41:42.673017625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:41:42.677273099Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-28T21:41:42.798304669Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:41:43.09754744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:41:44.221917706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55502"
level=debug ts=2026-04-28T21:41:44.765015918Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:41:46.902535148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:41:46.906431213Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:41:47.366848987Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:41:47.36690162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:41:47.372334485Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:41:48.098194293Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:48.098411649Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T21:41:50.889562341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:41:50.90008744Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:41:51.431648464Z caller=checkpoint.go:372 msg="writing series" size="101 kB"
level=info ts=2026-04-28T21:41:51.994222183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:41:51.999169598Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T21:41:53.098191649Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:41:54.765575448Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:41:57.233083744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:41:57.236899881Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:41:58.079019362Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:41:58.079383211Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:41:58.097228953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:41:58.097246236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:41:58.09724564Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:41:58.097254143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:41:58.098429925Z caller=checkpoint.go:372 msg="writing series" size="142 kB"
level=info ts=2026-04-28T21:41:59.869786139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:41:59.87811193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:42:01.432033901Z caller=checkpoint.go:372 msg="writing series" size="135 kB"
level=warn ts=2026-04-28T21:42:03.097895869Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:42:03.369751825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:42:03.374547445Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:42:03.426374794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:42:03.429962068Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:42:04.606887077Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:42:04.612983567Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:42:04.765339694Z caller=checkpoint.go:372 msg="writing series" size="69 kB"
level=debug ts=2026-04-28T21:42:07.540313223Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:42:08.097152539Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:08.098228251Z caller=checkpoint.go:372 msg="writing series" size="81 kB"
level=debug ts=2026-04-28T21:42:08.098243428Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
ts=2026-04-28T21:42:10.306979948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:40092"
level=debug ts=2026-04-28T21:42:11.450645479Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:42:11.450667981Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
ts=2026-04-28T21:42:12.800169114Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:42:13.09734196Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:13.117563876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:42:14.159314058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:42:14.163599967Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:42:14.765446054Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T21:42:17.324224507Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:42:17.332502538Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T21:42:18.098149275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:18.09823443Z caller=checkpoint.go:372 msg="writing series" size="196 kB"
level=info ts=2026-04-28T21:42:18.540565768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:42:18.549393646Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:42:18.549417052Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:42:18.549455159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:42:18.557479324Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:42:21.431870544Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:42:23.097155362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:42:23.333733492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:42:23.342041393Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:42:24.765184239Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:42:28.097092769Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:42:28.097116227Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:42:28.097124232Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:42:28.097142812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:28.09824385Z caller=checkpoint.go:372 msg="writing series" size="254 kB"
level=info ts=2026-04-28T21:42:30.142575807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:42:30.146695382Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:42:31.43232633Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-04-28T21:42:31.517740035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:42:31.521933502Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:42:31.755646578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:42:31.761889789Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:42:33.098165691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:34.766020679Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:42:34.809212876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:42:34.813184444Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:42:36.531963952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:42:36.537963589Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:42:37.953225498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:42:37.960859059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:42:38.097425652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:38.098507198Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:42:38.887250679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:42:38.890394745Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:42:39.609423763Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:42:39.612715618Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:42:40.121331459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="7.7 MB" avg_uncomp="7.7 MB" synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:42:40.126249901Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:42:41.431892328Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
ts=2026-04-28T21:42:42.802383643Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:42:43.09807581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:42:44.226305469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35610"
level=debug ts=2026-04-28T21:42:44.765144988Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=debug ts=2026-04-28T21:42:45.878092375Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:42:48.097968796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:42:48.097989233Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-28T21:42:49.917562759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:42:49.921729416Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T21:42:51.432509129Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-04-28T21:42:51.432529702Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-28T21:42:53.097997886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:42:53.956531734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:42:53.962645853Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:42:54.477379935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:42:54.485724314Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:42:54.765344827Z caller=checkpoint.go:372 msg="writing series" size="48 kB"
level=info ts=2026-04-28T21:42:56.407270893Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:42:56.412071288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:42:56.412091927Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:42:56.412113134Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:42:56.415708659Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:42:58.079094175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:42:58.079465957Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:42:58.097736425Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:42:58.09775743Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:42:58.097755589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:42:58.097766794Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:42:58.099898581Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-28T21:42:58.099909993Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-04-28T21:42:58.574599707Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:42:58.581662475Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:42:58.581680746Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:42:58.581704386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:42:58.5872294Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:42:58.911968309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:42:58.9151923Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:43:01.450286288Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:43:01.450330201Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-04-28T21:43:02.242895344Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:43:03.097765386Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:43:04.764664944Z caller=checkpoint.go:372 msg="writing series" size="89 kB"
level=info ts=2026-04-28T21:43:05.86088265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:43:05.87025257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:43:06.240874849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:43:06.249154928Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:43:08.097162091Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:43:08.098321865Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=debug ts=2026-04-28T21:43:11.432177144Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=debug ts=2026-04-28T21:43:12.310658088Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:43:12.804580003Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:43:13.098127063Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:43:14.764764067Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:43:15.294351189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:43:15.29828638Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:43:15.445551694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:43:15.450479707Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:43:15.525797672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:43:15.533085254Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:43:15.927892056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:43:15.931417679Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T21:43:18.09748623Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:43:18.09952189Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=info ts=2026-04-28T21:43:20.312644649Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:43:20.318029986Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:43:21.431753237Z caller=checkpoint.go:372 msg="writing series" size="212 kB"
level=debug ts=2026-04-28T21:43:21.431776248Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-28T21:43:23.097184248Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:24.566060748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:43:24.573136184Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:43:24.76572365Z caller=checkpoint.go:372 msg="writing series" size="139 kB"
level=info ts=2026-04-28T21:43:24.900167766Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:43:24.905197377Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:43:26.783599727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:43:26.787030255Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:43:28.097447867Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:43:28.09747278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:43:28.09748029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:43:28.097511046Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:43:28.09855497Z caller=checkpoint.go:384 msg="flushing series" totalSize="139 kB" series=1
level=info ts=2026-04-28T21:43:28.10292407Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000011.tmp new=/var/loki/wal/checkpoint.000011
level=info ts=2026-04-28T21:43:28.104662027Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.006815116s
level=info ts=2026-04-28T21:43:29.191661629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:43:29.197122966Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:43:30.354467138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:43:30.358095801Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T21:43:33.097510691Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:35.226127863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:43:35.230359935Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:43:38.097154185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:43:40.31177858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48156"
level=debug ts=2026-04-28T21:43:41.015328023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:43:41.285387122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:43:41.291128566Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-28T21:43:42.806571885Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=info ts=2026-04-28T21:43:42.923924009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:43:42.930672149Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T21:43:43.098149932Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:43.388817783Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:43:43.393115991Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:43:44.230611469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35156"
level=info ts=2026-04-28T21:43:45.331536914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:43:45.339720759Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:43:46.729463443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:43:46.738090412Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:43:48.097536495Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:48.617349502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:43:48.620958814Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:43:53.097571768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:53.158741249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=5 total_comp="12 MB" avg_comp="2.3 MB" total_uncomp="101 MB" avg_uncomp="20 MB" full=4 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:43:53.16683494Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:43:53.275004757Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:43:53.276655081Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T21:43:58.078995115Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:43:58.080034019Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:43:58.097626739Z caller=checkpoint.go:712 msg="starting checkpoint"
level=warn ts=2026-04-28T21:43:58.097634708Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:43:58.097657634Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:43:58.097667374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:43:58.097674369Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:43:58.097739762Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-28T21:43:58.097748065Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:43:58.097764983Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.031µs success=true
level=debug ts=2026-04-28T21:43:58.097948953Z caller=checkpoint.go:372 msg="writing series" size="4.2 kB"
level=debug ts=2026-04-28T21:44:01.431740792Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:44:01.915185184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:44:01.921010498Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T21:44:03.09781613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:03.69241588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:44:03.696093796Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:44:04.822158849Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-04-28T21:44:04.822184927Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=debug ts=2026-04-28T21:44:06.111209368Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:44:08.097735685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:44:08.099430291Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=info ts=2026-04-28T21:44:09.990308788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:44:09.996470644Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
ts=2026-04-28T21:44:10.313896241Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:57278"
level=debug ts=2026-04-28T21:44:11.43498171Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T21:44:11.435006171Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=2
ts=2026-04-28T21:44:12.808135033Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:44:13.09796841Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:14.054353057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:44:14.05981663Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-28T21:44:14.232054541Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41568"
level=debug ts=2026-04-28T21:44:14.766273962Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-04-28T21:44:15.694416858Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:44:15.697655564Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:44:17.747758374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:44:17.751740521Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:44:18.097997553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:44:18.09808417Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=info ts=2026-04-28T21:44:19.862240371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:44:19.86848887Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:44:21.431396062Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:44:23.097121813Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:24.37181311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:44:24.377224003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:44:24.768749882Z caller=checkpoint.go:372 msg="writing series" size="4.1 MB"
level=debug ts=2026-04-28T21:44:24.768780546Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=4
level=info ts=2026-04-28T21:44:24.858350722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:44:24.862575409Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:44:28.097431087Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:44:28.097454144Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:44:28.097461784Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:44:28.097510535Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:44:28.098834594Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:44:28.452224221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:44:28.456320524Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:44:29.947971545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:44:29.955525611Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:44:30.470024607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="34 MB" avg_uncomp="17 MB" full=1 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:44:30.476722902Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:44:30.574823066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:44:30.57898261Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:44:31.431467505Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:44:32.431743979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:44:32.435365549Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:44:32.898064629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:44:33.09820738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:33.635519014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:44:33.643761864Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:44:34.765757255Z caller=checkpoint.go:372 msg="writing series" size="963 kB"
level=debug ts=2026-04-28T21:44:34.765779699Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=3
level=debug ts=2026-04-28T21:44:36.99770372Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:44:38.097754677Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:44:38.098866714Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:44:41.432428217Z caller=checkpoint.go:372 msg="writing series" size="306 kB"
ts=2026-04-28T21:44:42.81011607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:44:43.097531508Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:43.791673438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:44:43.800541734Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:44:44.234785533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:50630"
level=debug ts=2026-04-28T21:44:44.765115258Z caller=checkpoint.go:372 msg="writing series" size="240 kB"
ts=2026-04-28T21:44:45.741299205Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41214"
level=info ts=2026-04-28T21:44:47.7130047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:44:47.718439674Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T21:44:48.097377118Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:44:48.098521549Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T21:44:49.904180827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:44:49.908216865Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:44:51.433287843Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-28T21:44:51.433307677Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=warn ts=2026-04-28T21:44:53.097414839Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:54.234101532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:44:54.239989582Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:44:54.765302865Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:44:58.079469806Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:44:58.079904576Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:44:58.097232101Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:44:58.097251302Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:44:58.097246126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:44:58.097259271Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:44:58.107852104Z caller=checkpoint.go:372 msg="writing series" size="8.3 MB"
level=debug ts=2026-04-28T21:44:58.10786923Z caller=checkpoint.go:384 msg="flushing series" totalSize="8.5 MB" series=2
level=info ts=2026-04-28T21:44:59.797192096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:44:59.801497733Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:45:01.432637755Z caller=checkpoint.go:372 msg="writing series" size="948 kB"
level=warn ts=2026-04-28T21:45:03.098118023Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:45:03.110224921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:45:03.113936069Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:45:03.932573517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:45:03.939432676Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:45:04.764849049Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-04-28T21:45:04.91449541Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:45:05.038694035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:45:05.044040293Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T21:45:08.097282911Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:08.099046423Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:45:08.099066287Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
ts=2026-04-28T21:45:10.318597754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33372"
level=info ts=2026-04-28T21:45:10.838475537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:45:10.847891193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:45:11.432321287Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:45:12.812304537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:45:13.098000685Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:45:13.772823454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:45:13.778318278Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:45:14.201715947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:45:14.205086059Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:45:14.765221553Z caller=checkpoint.go:372 msg="writing series" size="151 kB"
level=debug ts=2026-04-28T21:45:16.015454625Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:45:18.098068515Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:18.098067303Z caller=checkpoint.go:372 msg="writing series" size="6.7 kB"
level=info ts=2026-04-28T21:45:19.828890453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:45:19.836071309Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:45:21.431961289Z caller=checkpoint.go:372 msg="writing series" size="213 kB"
level=warn ts=2026-04-28T21:45:23.097229997Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:45:23.734669734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:45:23.744437363Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:45:23.744460882Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:45:23.744491227Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:45:23.752215111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:45:24.766007484Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-28T21:45:24.766029784Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=5
level=info ts=2026-04-28T21:45:26.125345512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:45:26.12886485Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:45:26.397532539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:45:26.404867524Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:45:28.097218676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:45:28.097247723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:45:28.097255068Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:45:28.097243943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:28.098367049Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:45:29.352235538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:45:29.359536033Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:45:30.456879671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:45:30.461433056Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:45:31.431642816Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T21:45:33.097210324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:34.76546418Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T21:45:37.436457979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:45:37.448022413Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:45:38.097918527Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:38.098137997Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:45:38.533509926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:45:38.538246806Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:45:39.208849088Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T21:45:40.320819018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47798"
level=debug ts=2026-04-28T21:45:41.432331835Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T21:45:41.576674384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:45:41.580963147Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:45:41.580987847Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:45:41.581020126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:45:41.585162888Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-28T21:45:42.814345949Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:45:43.097649948Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:44.765045142Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T21:45:45.058947335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:45:45.066046739Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-28T21:45:45.745097062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52352"
level=info ts=2026-04-28T21:45:47.578827053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:45:47.582663835Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T21:45:48.09797652Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:48.0983777Z caller=checkpoint.go:372 msg="writing series" size="514 kB"
level=debug ts=2026-04-28T21:45:48.098391849Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-28T21:45:50.119810853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:45:50.126286255Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:45:50.190587775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:45:50.195745351Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:45:50.195766453Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:45:50.195791042Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:45:50.201016163Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:45:51.432295932Z caller=checkpoint.go:372 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T21:45:53.097436291Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:45:53.110525577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:45:53.114531134Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:45:54.765027617Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:45:55.56273746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:45:55.566789933Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:45:56.962871448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:45:56.968883116Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:45:58.078985866Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:45:58.079861611Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-04-28T21:45:58.081100301Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T21:45:58.081143539Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974902 n=1
level=debug ts=2026-04-28T21:45:58.08118345Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974902 n=1
level=info ts=2026-04-28T21:45:58.09753274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:45:58.097533402Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:45:58.097555036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:45:58.097562777Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:45:58.098721951Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=debug ts=2026-04-28T21:46:01.431823082Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T21:46:02.046625937Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:46:02.052289321Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:46:03.097973009Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:46:03.376601365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:46:03.384094176Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:46:04.765226809Z caller=checkpoint.go:372 msg="writing series" size="497 kB"
level=debug ts=2026-04-28T21:46:04.955543274Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:46:06.901569776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:46:06.909124484Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:46:08.008346498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:46:08.012996279Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:46:08.097888926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:08.098344725Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-04-28T21:46:08.098359375Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-04-28T21:46:10.738248138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:46:10.746597399Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:46:11.432035155Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-04-28T21:46:12.815694926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:46:13.09731674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:46:13.837801225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:46:13.842576707Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:46:14.038702002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:46:14.043678513Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:46:14.043714189Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:14.04374011Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:46:14.046770497Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:46:14.765028544Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
ts=2026-04-28T21:46:15.747229329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38328"
level=debug ts=2026-04-28T21:46:18.090289954Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:46:18.097400348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:18.098798168Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:46:19.228859024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:46:19.233400897Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:46:20.942731107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:46:20.948355339Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:46:21.431643401Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:46:23.097683362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:24.765517Z caller=checkpoint.go:372 msg="writing series" size="570 kB"
level=debug ts=2026-04-28T21:46:24.765543553Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T21:46:27.232068491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:46:27.242383928Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:46:28.097228048Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:46:28.097256234Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:46:28.097246842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:46:28.097265482Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:46:28.098400646Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:46:29.04003664Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:46:29.044109852Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:46:30.852468865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:46:30.857237735Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:46:31.431925344Z caller=checkpoint.go:372 msg="writing series" size="385 kB"
level=info ts=2026-04-28T21:46:31.876797147Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:46:31.88030396Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:46:31.880323422Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:31.880343436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:46:31.883131332Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:46:32.227971902Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:46:32.97016496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:46:32.975309014Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:46:33.097846272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:34.765289169Z caller=checkpoint.go:372 msg="writing series" size="232 kB"
level=debug ts=2026-04-28T21:46:38.097989558Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-04-28T21:46:38.098000051Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:46:40.255748444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:46:40.261278697Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:46:40.261298676Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:40.261320293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:46:40.265351017Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-28T21:46:40.32512361Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:60524"
level=info ts=2026-04-28T21:46:40.629411781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:46:40.635430336Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:46:41.431536996Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:46:41.431558909Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
ts=2026-04-28T21:46:42.816838979Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:46:43.098135928Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:44.765643614Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:46:44.798843506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:46:44.802385933Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:46:45.513500697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:46:45.517261354Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:46:45.517282249Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:45.517302598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:46:45.520320964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-28T21:46:45.749254712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56766"
level=info ts=2026-04-28T21:46:46.902531142Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:46:46.905895135Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:46:46.905914526Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:46.905937488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:46:46.908650386Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T21:46:48.097518189Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:48.098861868Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-04-28T21:46:51.431824343Z caller=checkpoint.go:372 msg="writing series" size="104 kB"
level=info ts=2026-04-28T21:46:51.567981243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:46:51.573155433Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T21:46:53.098136834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:46:54.54860164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:46:54.552721866Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T21:46:54.552747702Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:46:54.55277591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:46:54.556352801Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:46:54.765026157Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:46:57.330963807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:46:57.334306898Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:46:58.079195268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:46:58.079546331Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:46:58.097927431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:46:58.097951039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:46:58.097969311Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:46:58.097983492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:46:58.098026778Z caller=checkpoint.go:372 msg="writing series" size="150 kB"
level=info ts=2026-04-28T21:46:59.194309678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:46:59.198151437Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:46:59.676550831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:46:59.683634703Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:47:01.432373768Z caller=checkpoint.go:372 msg="writing series" size="144 kB"
level=warn ts=2026-04-28T21:47:03.097679277Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:03.54802356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:47:04.765721591Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=info ts=2026-04-28T21:47:05.307364114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:47:05.311326219Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:47:05.924630425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:47:05.931663652Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:47:06.648918724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:47:08.097921435Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:08.097969643Z caller=checkpoint.go:372 msg="writing series" size="84 kB"
level=debug ts=2026-04-28T21:47:08.097984237Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-28T21:47:10.256885903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:47:10.26465513Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:47:10.474430431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:47:10.47839501Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:47:11.013730839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:47:11.017042237Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T21:47:11.017061849Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:47:11.017082667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:47:11.019838414Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:47:11.450753055Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:47:11.450779598Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=info ts=2026-04-28T21:47:11.969610605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:47:11.973888842Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T21:47:12.818061539Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:47:13.097815396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:47:13.457306288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:47:13.460690212Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:47:13.741266933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:47:13.746134425Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-04-28T21:47:13.746154167Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:47:13.746175865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:47:13.750425576Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:47:13.793667481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:47:13.797860485Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:47:14.764874064Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-04-28T21:47:15.75173058Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52712"
level=warn ts=2026-04-28T21:47:18.097419013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:18.098706245Z caller=checkpoint.go:372 msg="writing series" size="197 kB"
level=info ts=2026-04-28T21:47:19.518800091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:47:19.522500846Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T21:47:21.431637045Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:47:23.097208765Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:47:23.925669395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:47:23.935013469Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:47:24.765479703Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T21:47:25.146993603Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:47:25.15208892Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:47:28.098021993Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:47:28.09804635Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:47:28.098056566Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:47:28.098059604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:28.098312321Z caller=checkpoint.go:372 msg="writing series" size="284 kB"
level=info ts=2026-04-28T21:47:30.508314541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:47:30.514400177Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:47:30.930497532Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:47:30.934742619Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:47:31.432290911Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-04-28T21:47:33.027118292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:47:33.032365915Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T21:47:33.097855588Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:34.749871086Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:47:34.764934221Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:47:36.887979138Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:47:36.892033034Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T21:47:38.097919221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:38.097950297Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-28T21:47:41.432085931Z caller=checkpoint.go:372 msg="writing series" size="53 kB"
level=info ts=2026-04-28T21:47:42.484683149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:47:42.490306061Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-28T21:47:42.819859897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:47:43.097869779Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:47:43.12004268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:47:43.123756994Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:47:43.167822208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:47:43.172240039Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:47:44.765101417Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-04-28T21:47:47.841300324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:47:47.850497814Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:47:48.098036792Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=warn ts=2026-04-28T21:47:48.098091523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:47:51.140763792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:47:51.144259764Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:47:51.431979025Z caller=checkpoint.go:372 msg="writing series" size="238 kB"
level=debug ts=2026-04-28T21:47:51.432002026Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=warn ts=2026-04-28T21:47:53.097291107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:47:53.54815255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:47:53.556297216Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:47:54.006407314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:47:54.011336715Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:47:54.765440618Z caller=checkpoint.go:372 msg="writing series" size="50 kB"
level=info ts=2026-04-28T21:47:57.240554429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:47:57.245151555Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:47:58.07869235Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:47:58.079871973Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T21:47:58.097484966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:47:58.097504346Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:47:58.097511099Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:47:58.097507039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:47:58.100703091Z caller=checkpoint.go:372 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-28T21:47:58.100715158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=2
level=debug ts=2026-04-28T21:48:00.198275221Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:48:00.355743254Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:48:00.357719717Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T21:48:01.447578666Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-28T21:48:01.447605353Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=1
level=info ts=2026-04-28T21:48:02.686221293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:48:02.690329701Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:48:03.097544157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:04.764764845Z caller=checkpoint.go:372 msg="writing series" size="121 kB"
level=info ts=2026-04-28T21:48:05.299100261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:48:05.304251974Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:48:06.588568365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:48:06.597068827Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:48:06.597090039Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:48:06.597112581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:48:06.604756002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:48:07.517457474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:48:07.520941503Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:48:08.028471565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:48:08.03293364Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T21:48:08.097555403Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:08.098672859Z caller=checkpoint.go:372 msg="writing series" size="654 B"
ts=2026-04-28T21:48:10.331029989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48792"
level=debug ts=2026-04-28T21:48:11.431630851Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
ts=2026-04-28T21:48:12.822470608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:48:13.097165739Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:14.765607419Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:48:17.605383619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:48:17.611732881Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:48:17.611753968Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:48:17.611778627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:48:17.617070933Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:48:18.09768096Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:18.099605434Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:48:18.099639963Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-28T21:48:19.474131256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:48:19.481281568Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:48:20.775092962Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:48:20.783377971Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:48:21.432353235Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=warn ts=2026-04-28T21:48:23.09808111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:24.765643899Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=debug ts=2026-04-28T21:48:25.623966707Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:48:27.988090004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:48:27.993208592Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:48:28.097722958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:48:28.097747465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:48:28.097754811Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:48:28.097753059Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:28.098828111Z caller=checkpoint.go:384 msg="flushing series" totalSize="358 kB" series=2
level=info ts=2026-04-28T21:48:28.10747153Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000012.tmp new=/var/loki/wal/checkpoint.000012
level=info ts=2026-04-28T21:48:28.109178108Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.011297084s
level=info ts=2026-04-28T21:48:30.41839623Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:48:30.422948862Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:48:33.098054853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:48:36.388664352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:48:36.395685697Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:48:37.115890304Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:48:37.121259455Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T21:48:38.097435126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:48:38.275962712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:48:38.279899315Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:48:40.274133332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:48:40.277569945Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:48:41.843218396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:48:41.850463913Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:48:41.850486922Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:48:41.850523582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:48:41.857136317Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:48:42.824719194Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:48:43.097421645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:48:43.885351432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:48:43.892843026Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T21:48:44.251526085Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:54870"
level=debug ts=2026-04-28T21:48:45.53274385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:48:48.09725276Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:48:49.43806081Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:48:49.443720508Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:48:49.642186752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:48:49.645470542Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:48:49.929369464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:48:49.934143296Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:48:50.927132332Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:48:50.934238325Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:48:51.580652922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:48:51.5853381Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:48:52.943769352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:48:52.951024165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:48:53.09774556Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:48:55.773167695Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:48:58.010058002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:48:58.016201638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:48:58.07912591Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:48:58.080146973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:48:58.096944956Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:48:58.097076572Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-28T21:48:58.097085142Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:48:58.097101736Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=12.36µs success=true
level=warn ts=2026-04-28T21:48:58.09718134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:48:58.097198835Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:48:58.097237392Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:48:58.097246084Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:48:58.097311093Z caller=checkpoint.go:372 msg="writing series" size="4.4 kB"
level=debug ts=2026-04-28T21:49:01.430693706Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:49:02.066399581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:49:02.07179595Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:49:03.097452134Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:04.81023516Z caller=checkpoint.go:372 msg="writing series" size="60 MB"
level=debug ts=2026-04-28T21:49:04.810261788Z caller=checkpoint.go:384 msg="flushing series" totalSize="60 MB" series=3
level=warn ts=2026-04-28T21:49:08.097493021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:08.098017188Z caller=checkpoint.go:372 msg="writing series" size="635 kB"
level=debug ts=2026-04-28T21:49:11.434672841Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T21:49:11.434692454Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=2
level=info ts=2026-04-28T21:49:11.560741166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:49:11.569299974Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-28T21:49:12.826399334Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:49:13.097568016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:14.765115544Z caller=checkpoint.go:372 msg="writing series" size="327 kB"
level=info ts=2026-04-28T21:49:15.853247412Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:49:15.861096338Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:49:16.304332192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:49:16.310606071Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:49:18.097532018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:18.097731634Z caller=checkpoint.go:372 msg="writing series" size="141 kB"
level=debug ts=2026-04-28T21:49:20.139077339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:49:21.431673944Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T21:49:23.097397207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:49:23.907787051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:49:23.914266086Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:49:24.769268663Z caller=checkpoint.go:372 msg="writing series" size="4.2 MB"
level=debug ts=2026-04-28T21:49:24.76931262Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.7 MB" series=4
level=info ts=2026-04-28T21:49:25.464136591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:49:25.468672373Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:49:25.938179328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:49:25.944200591Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:49:28.097185561Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:49:28.097211681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:49:28.09721933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:49:28.097298943Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:28.097607635Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:49:28.438740587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:49:28.44417827Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:49:29.175111093Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:49:31.431662036Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T21:49:32.141514281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:49:32.150083813Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:49:32.587190596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:49:32.590923067Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:49:32.840544128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:49:32.844983298Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:49:33.098091237Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:34.76498693Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:49:34.765008622Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=warn ts=2026-04-28T21:49:38.097252169Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:38.098417125Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T21:49:41.431629904Z caller=checkpoint.go:372 msg="writing series" size="357 kB"
level=info ts=2026-04-28T21:49:42.70274909Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:49:42.709591152Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
ts=2026-04-28T21:49:42.828637312Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:49:43.097906653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:44.765080878Z caller=checkpoint.go:372 msg="writing series" size="267 kB"
level=info ts=2026-04-28T21:49:45.2614154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:49:45.266641851Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:49:45.266661345Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:49:45.266685314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:49:45.271293594Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:49:45.468843153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:49:45.472237759Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:49:48.097997221Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:48.097998259Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T21:49:48.682959136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:49:48.686305099Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T21:49:48.949060763Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:49:50.876182934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:49:50.884932664Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:49:51.433472809Z caller=checkpoint.go:372 msg="writing series" size="2.5 MB"
level=debug ts=2026-04-28T21:49:51.433492118Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-04-28T21:49:51.474352397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:49:51.479519134Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:49:53.097395186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:54.765099329Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:49:55.949547515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:49:55.953952637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:49:58.07896294Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:49:58.079869699Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:49:58.097126274Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:49:58.097148051Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:49:58.097157492Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:49:58.097234989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:49:58.108063893Z caller=checkpoint.go:372 msg="writing series" size="9.7 MB"
level=debug ts=2026-04-28T21:49:58.108081551Z caller=checkpoint.go:384 msg="flushing series" totalSize="9.9 MB" series=2
level=debug ts=2026-04-28T21:50:01.431848641Z caller=checkpoint.go:372 msg="writing series" size="747 kB"
level=warn ts=2026-04-28T21:50:03.098213753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:04.764036359Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:50:06.254894598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:50:06.264241865Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:50:08.097342844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:08.097997422Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:50:08.098013361Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-04-28T21:50:08.718149263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:50:08.72207532Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:50:10.254081882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:50:10.261727932Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:50:10.261748525Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:50:10.261773584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:50:10.268073273Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:50:11.431099668Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-04-28T21:50:11.770127069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:50:11.774477825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T21:50:12.830706316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:50:13.097334232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:14.764998496Z caller=checkpoint.go:372 msg="writing series" size="149 kB"
level=info ts=2026-04-28T21:50:15.099283007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:50:15.106087886Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:50:18.097900693Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:18.09795368Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-28T21:50:19.521272009Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:50:20.93032309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:50:21.431840413Z caller=checkpoint.go:372 msg="writing series" size="226 kB"
level=info ts=2026-04-28T21:50:22.080792887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:50:22.084643677Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:50:23.097280212Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:50:24.25889107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:50:24.262862376Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T21:50:24.765233064Z caller=checkpoint.go:372 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-28T21:50:24.765259269Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=5
level=info ts=2026-04-28T21:50:24.837847543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:50:24.845180528Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:50:25.505331727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:50:25.510201373Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:50:26.006605419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:50:26.01439524Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:50:27.231542282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:50:27.234949383Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:50:28.097139035Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:50:28.097165933Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:50:28.097174631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:50:28.097166426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:28.098310857Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:50:28.261749145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:50:28.266912622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:50:28.541005667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:50:28.54490002Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:50:31.431635192Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-04-28T21:50:31.808741777Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:50:31.81654134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:50:32.623121514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:50:32.626470887Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T21:50:33.097401332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:34.764335879Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T21:50:36.207830426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:50:36.211085768Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:50:38.097734019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:38.097950004Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T21:50:41.431013988Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T21:50:42.832475823Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:50:43.097816805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:44.764273161Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-04-28T21:50:45.764260328Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33180"
level=warn ts=2026-04-28T21:50:48.097663301Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:48.098122282Z caller=checkpoint.go:372 msg="writing series" size="459 kB"
level=debug ts=2026-04-28T21:50:48.098137601Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-04-28T21:50:48.882851102Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:50:49.554841044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:50:49.564143829Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:50:51.431744668Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T21:50:53.098001612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:54.765008995Z caller=checkpoint.go:372 msg="writing series" size="936 B"
level=info ts=2026-04-28T21:50:57.057292682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:50:57.065784402Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:50:58.078987255Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:50:58.07904432Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:50:58.097808638Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:50:58.097828458Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:50:58.097836368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:50:58.097858695Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:50:58.097919712Z caller=checkpoint.go:372 msg="writing series" size="137 kB"
level=info ts=2026-04-28T21:50:59.579414306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:50:59.583131111Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:50:59.865792374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:50:59.872143992Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:51:01.43091046Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T21:51:01.994740804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:51:02.000487581Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:51:03.097892506Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:04.765239675Z caller=checkpoint.go:372 msg="writing series" size="520 kB"
level=warn ts=2026-04-28T21:51:08.097185501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:08.09903095Z caller=checkpoint.go:372 msg="writing series" size="609 kB"
level=debug ts=2026-04-28T21:51:08.099045976Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=6
level=info ts=2026-04-28T21:51:08.271535744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:51:08.276810734Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:51:08.690047566Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:51:08.693712871Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:51:08.875914236Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:51:08.880754407Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:51:11.43145295Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-04-28T21:51:11.903216656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:51:11.906487875Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-28T21:51:12.834480234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:51:13.098059503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:13.755519123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T21:51:14.262514406Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:59140"
level=debug ts=2026-04-28T21:51:14.764448681Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=warn ts=2026-04-28T21:51:18.097598799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:18.098166478Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=info ts=2026-04-28T21:51:18.995385877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:51:18.999657167Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:51:19.79685843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:51:19.801062627Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:51:21.430966867Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=debug ts=2026-04-28T21:51:22.046764424Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:51:23.097943931Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:51:23.210288277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:51:23.21485467Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:51:23.413580321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:51:23.420665161Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:51:24.764922547Z caller=checkpoint.go:372 msg="writing series" size="577 kB"
level=debug ts=2026-04-28T21:51:24.764949074Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T21:51:26.133409791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:51:26.137280385Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:51:27.271214729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:51:27.275891233Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:51:28.097133891Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:51:28.097160257Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:51:28.097177405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:51:28.097220488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:28.098418201Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T21:51:29.530839682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:51:29.539040792Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:51:31.431802694Z caller=checkpoint.go:372 msg="writing series" size="492 kB"
level=info ts=2026-04-28T21:51:32.214380203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:51:32.219530114Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:51:33.097530541Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:34.764848003Z caller=checkpoint.go:372 msg="writing series" size="318 kB"
level=warn ts=2026-04-28T21:51:38.097502638Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:38.097550466Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
ts=2026-04-28T21:51:40.344958072Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:38394"
level=debug ts=2026-04-28T21:51:41.431644939Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T21:51:41.431667728Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
ts=2026-04-28T21:51:42.836682999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:51:43.098100699Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:44.764634403Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:51:45.76788762Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41378"
level=warn ts=2026-04-28T21:51:48.097772634Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:48.098093325Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-04-28T21:51:50.088180849Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:51:51.431149186Z caller=checkpoint.go:372 msg="writing series" size="107 kB"
level=info ts=2026-04-28T21:51:51.75294683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:51:51.759783963Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:51:52.701663628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:51:52.711279294Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:51:53.098237952Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:51:54.435093934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:51:54.438807421Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T21:51:54.764880401Z caller=checkpoint.go:372 msg="writing series" size="14 kB"
level=info ts=2026-04-28T21:51:55.135833377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:51:55.142151119Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:51:56.644876364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:51:56.648267996Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:51:58.079061476Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:51:58.079094795Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:51:58.097961636Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:51:58.09798541Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:51:58.097993509Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:51:58.098015626Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:51:58.098097737Z caller=checkpoint.go:372 msg="writing series" size="155 kB"
level=info ts=2026-04-28T21:51:58.909208396Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:51:58.917786346Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:51:59.905479591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:51:59.908823371Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:52:01.431267235Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=warn ts=2026-04-28T21:52:03.097504313Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:52:03.700572638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:52:03.703988095Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:52:04.764404702Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-04-28T21:52:05.550846164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:52:05.555847977Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:52:06.162667303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:52:06.166856568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:52:06.775735927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:52:06.779937104Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:52:06.779958648Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:52:06.779981141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:52:06.783469731Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:52:08.097397207Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:08.097481129Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=debug ts=2026-04-28T21:52:08.097497818Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=8
level=info ts=2026-04-28T21:52:09.722971648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:52:09.730705098Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T21:52:10.34728107Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34026"
level=debug ts=2026-04-28T21:52:11.452712523Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:52:11.452746844Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=1
level=debug ts=2026-04-28T21:52:11.933837512Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:52:11.935757807Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T21:52:12.838696875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
ts=2026-04-28T21:52:12.839402448Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=error ts=2026-04-28T21:52:12.842506526Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T21:52:13.098016088Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:52:13.354739263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:52:13.358186526Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:52:14.764661181Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=debug ts=2026-04-28T21:52:17.241983806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:52:17.494677485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:52:17.503069005Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:52:17.503090821Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:52:17.503118732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:52:17.510611051Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:52:18.097672131Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:18.097809223Z caller=checkpoint.go:372 msg="writing series" size="198 kB"
level=info ts=2026-04-28T21:52:19.507182515Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:52:19.512832207Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:52:19.512854602Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:52:19.512877494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:52:19.51792346Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:52:21.431462399Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T21:52:23.097643478Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:52:24.310667093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:52:24.31472642Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:52:24.764333662Z caller=checkpoint.go:372 msg="writing series" size="17 kB"
level=info ts=2026-04-28T21:52:28.097161742Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:52:28.097191076Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:52:28.09720103Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:52:28.097193286Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:28.097503686Z caller=checkpoint.go:372 msg="writing series" size="340 kB"
level=info ts=2026-04-28T21:52:29.665669963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:52:29.671629837Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:52:31.431544129Z caller=checkpoint.go:372 msg="writing series" size="148 kB"
level=info ts=2026-04-28T21:52:32.082586496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:52:32.092284003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:52:33.098074744Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:34.764781522Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:52:35.060568208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:52:35.064281911Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:52:35.9258272Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:52:35.930575901Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:52:38.097187862Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:38.098326519Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:52:40.35093975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:52:40.358232309Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:52:41.431468369Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-04-28T21:52:42.841482337Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:52:43.097496653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:52:44.268281916Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57854"
level=debug ts=2026-04-28T21:52:44.764985906Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
level=info ts=2026-04-28T21:52:44.972877445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:52:44.976243885Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:52:44.976265575Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:52:44.976284942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:52:44.979250902Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:52:45.978292196Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:52:48.097916024Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:48.097944179Z caller=checkpoint.go:372 msg="writing series" size="38 kB"
level=info ts=2026-04-28T21:52:48.233320438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:52:48.238407235Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:52:48.238429915Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:52:48.238451107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:52:48.241351797Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:52:49.310238804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=20 total_comp="58 MB" avg_comp="2.9 MB" total_uncomp="477 MB" avg_uncomp="24 MB" full=20 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:52:49.317952797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 20, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:52:51.431848314Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=debug ts=2026-04-28T21:52:51.431869378Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=info ts=2026-04-28T21:52:52.814857619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:52:52.821660994Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:52:53.09796284Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:54.764877773Z caller=checkpoint.go:372 msg="writing series" size="51 kB"
level=info ts=2026-04-28T21:52:55.119106473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:52:55.125441585Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:52:56.010290037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:52:56.018855537Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:52:56.85814419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:52:56.861737476Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:52:58.079072268Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:52:58.079085847Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:52:58.0977676Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:52:58.097792196Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:52:58.097801093Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:52:58.097841633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:52:58.099955901Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-28T21:52:58.099968114Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=2
level=info ts=2026-04-28T21:52:58.54849157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:52:58.556464879Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:52:59.521293496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:53:01.352843063Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:53:01.359107554Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:53:01.453102805Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-04-28T21:53:01.453132436Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=1
level=info ts=2026-04-28T21:53:02.087358872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:53:02.090949833Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:53:02.867913117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:53:02.873435632Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:53:03.097224076Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:03.127291635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:53:03.131474869Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:53:03.239717802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:53:03.243726915Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:53:04.059338293Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:53:04.063848731Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:53:04.764404665Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-04-28T21:53:06.621388057Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:53:06.625050631Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:53:06.625072066Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:06.625092657Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:53:06.628035783Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T21:53:08.097641065Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:53:08.097675379Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=debug ts=2026-04-28T21:53:11.431221899Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
ts=2026-04-28T21:53:12.843610537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:53:13.097971842Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:53:13.840221726Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:53:14.76402271Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:53:16.145645188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:53:16.15146206Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:53:16.151492629Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:16.151519977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:53:16.155379404Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T21:53:18.097982017Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:53:18.098430581Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:53:18.098448874Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=debug ts=2026-04-28T21:53:21.431638162Z caller=checkpoint.go:372 msg="writing series" size="224 kB"
level=warn ts=2026-04-28T21:53:23.098030697Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:23.615066115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:53:23.620380669Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:53:24.361399378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:53:24.365898585Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T21:53:24.365922798Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:24.365951436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:53:24.370121825Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:53:24.76417026Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-04-28T21:53:25.885535136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=3 total_comp="8.0 MB" avg_comp="2.7 MB" total_uncomp="52 MB" avg_uncomp="18 MB" full=2 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:53:25.892774754Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:53:27.957974901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:53:27.961887567Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:53:28.097439139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:53:28.097440777Z caller=checkpoint.go:384 msg="flushing series" totalSize="376 kB" series=2
level=debug ts=2026-04-28T21:53:28.097464976Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T21:53:28.09746488Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:28.097474701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:53:28.134062898Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000013.tmp new=/var/loki/wal/checkpoint.000013
level=info ts=2026-04-28T21:53:28.135609845Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.038374192s
level=info ts=2026-04-28T21:53:29.993325172Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:53:29.997554108Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:53:33.097405558Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:34.902205887Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:53:34.906587718Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:53:35.455388714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:53:35.462103305Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T21:53:38.097665803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:39.780133043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:53:39.7843993Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:53:40.213017064Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:53:41.430349699Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="506 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:53:41.439325541Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T21:53:42.8462278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:53:43.097772272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:53:44.271502152Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:49364"
level=info ts=2026-04-28T21:53:44.46827789Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:53:44.475809072Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:53:47.923976668Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:53:48.098071611Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:51.907569776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:53:51.911610168Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T21:53:51.911649946Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:51.911672182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:53:51.915401786Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:53:53.097998727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:53.11116101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:53:53.11628182Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-04-28T21:53:53.116304814Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:53.116332263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:53:53.121171058Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:53:54.715678248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:53:54.723332472Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T21:53:55.698438029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:53:55.703001856Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:53:58.079065955Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:53:58.080189615Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:53:58.097153647Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:53:58.097179072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:53:58.097187327Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:53:58.097194633Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:53:58.097199895Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:53:58.097345952Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-28T21:53:58.097355826Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:53:58.097372189Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.824µs success=true
level=debug ts=2026-04-28T21:53:58.097591911Z caller=checkpoint.go:372 msg="writing series" size="4.7 kB"
level=info ts=2026-04-28T21:53:58.182751051Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:53:58.188605414Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:53:59.029847741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:53:59.033366259Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:53:59.275441069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:53:59.279314617Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:53:59.279337359Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:53:59.279359795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:53:59.282472756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:53:59.283508628Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:53:59.285363194Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:54:01.083236268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:54:01.088499596Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:54:01.351104878Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T21:54:02.917580846Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:54:02.921545391Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:54:03.097483807Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:54:03.566306314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:54:03.572088276Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:54:03.959879768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:54:03.964823354Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:54:04.680685373Z caller=checkpoint.go:372 msg="writing series" size="64 MB"
level=debug ts=2026-04-28T21:54:04.68072295Z caller=checkpoint.go:384 msg="flushing series" totalSize="64 MB" series=3
level=debug ts=2026-04-28T21:54:06.011997264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:54:07.858314575Z caller=checkpoint.go:372 msg="writing series" size="619 kB"
level=warn ts=2026-04-28T21:54:08.097603741Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:11.115809927Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T21:54:11.115847461Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.3 MB" series=2
ts=2026-04-28T21:54:12.849455695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T21:54:13.097474079Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:14.363904883Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-04-28T21:54:14.398169683Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:54:14.407422603Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-28T21:54:15.78045858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51448"
level=debug ts=2026-04-28T21:54:17.616512426Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T21:54:18.097241425Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:54:19.05148747Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:54:19.060695408Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:54:19.255426164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:54:19.25932547Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:54:20.868844724Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T21:54:21.259310673Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:54:21.26330981Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:54:21.993852678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=22 total_comp="61 MB" avg_comp="2.8 MB" total_uncomp="506 MB" avg_uncomp="23 MB" full=21 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:54:22.002370114Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 22, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:54:23.09745235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:24.127699401Z caller=checkpoint.go:372 msg="writing series" size="4.4 MB"
level=debug ts=2026-04-28T21:54:24.127729161Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.8 MB" series=4
level=debug ts=2026-04-28T21:54:26.067376905Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:54:27.375910016Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:54:28.097202296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:54:28.097237229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:54:28.097248042Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:54:28.0973058Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:54:29.675369247Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:54:29.682107276Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:54:30.628589293Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:54:33.09783867Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:33.383960234Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:54:33.882570473Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:54:33.882594578Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.5 MB" series=3
level=info ts=2026-04-28T21:54:36.259777852Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:54:36.26535074Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:54:37.134177464Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-04-28T21:54:37.6635904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:54:37.671806933Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:54:38.097508396Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:54:39.633851772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:54:39.637738775Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-28T21:54:40.357504791Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55484"
level=debug ts=2026-04-28T21:54:40.386934253Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-04-28T21:54:41.407392019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:54:41.412151601Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:54:42.51220718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:54:42.520435641Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:54:42.520460834Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:54:42.520487574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:54:42.528525304Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T21:54:42.852885457Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:54:43.097896811Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:43.640280881Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=info ts=2026-04-28T21:54:46.561251511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:54:46.566787859Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:54:46.892906232Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T21:54:48.098055736Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:50.150354374Z caller=checkpoint.go:372 msg="writing series" size="2.6 MB"
level=debug ts=2026-04-28T21:54:50.150383804Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=warn ts=2026-04-28T21:54:53.097385624Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:54:53.399722937Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:54:54.063744024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:54:54.068686538Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:54:54.395832205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:54:54.399787165Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:54:54.508676601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:54:54.512063901Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:54:54.675658681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:54:54.679324864Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:54:56.214329255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:54:56.219437035Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:54:56.666779683Z caller=checkpoint.go:372 msg="writing series" size="12 MB"
level=debug ts=2026-04-28T21:54:56.666831819Z caller=checkpoint.go:384 msg="flushing series" totalSize="12 MB" series=2
level=info ts=2026-04-28T21:54:58.078987252Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:54:58.079770397Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:54:58.097080367Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:54:58.097104689Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:54:58.097112313Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:54:58.097229185Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:54:58.425867462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:54:58.431514694Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:54:58.549829317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:54:58.557546855Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:54:59.905739486Z caller=checkpoint.go:372 msg="writing series" size="872 kB"
level=info ts=2026-04-28T21:55:01.46789395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:55:01.475917151Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:55:02.76302047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:55:02.766764434Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T21:55:03.09734748Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:55:03.15865859Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=debug ts=2026-04-28T21:55:06.411764689Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T21:55:06.411795349Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=debug ts=2026-04-28T21:55:07.82125669Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:55:07.821321886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="198 B" avg_comp="198 B" total_uncomp="615 B" avg_uncomp="615 B" idle=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:55:07.825888243Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=warn ts=2026-04-28T21:55:08.097153467Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:55:09.664059049Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T21:55:12.854581397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=debug ts=2026-04-28T21:55:12.917054633Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T21:55:13.097433846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:14.527634312Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:55:14.537171349Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:55:14.537196227Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:55:14.537240725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:55:14.546378009Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:55:16.008425358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:55:16.01611496Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:55:16.01614022Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:55:16.016165771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:55:16.023472921Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:55:16.170344669Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=warn ts=2026-04-28T21:55:18.097818294Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:19.358516934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:55:19.368078863Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T21:55:19.423226745Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-28T21:55:22.676837134Z caller=checkpoint.go:372 msg="writing series" size="242 kB"
level=warn ts=2026-04-28T21:55:23.097231241Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:23.284350325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:55:23.28808319Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:55:23.957461335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:55:23.966840946Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:55:25.929601706Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T21:55:25.929649742Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=6
level=debug ts=2026-04-28T21:55:26.736972894Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:55:28.097778884Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:28.098004153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:55:28.098092742Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:55:28.09810432Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:55:29.182776391Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T21:55:29.394364294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:55:29.400598428Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:55:32.435165678Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=info ts=2026-04-28T21:55:32.601880194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:55:32.608347567Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:55:33.054762636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:55:33.063207323Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:55:33.098144049Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:35.253800558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:55:35.266656419Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:55:35.688432747Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T21:55:36.90681175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:55:36.911221136Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:55:38.09719595Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:55:38.941528901Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T21:55:39.093302872Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:55:42.194993112Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T21:55:42.518355522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:55:42.526912439Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-28T21:55:42.856531681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:55:43.097515672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:45.274684535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:55:45.279112922Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:55:45.447454807Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T21:55:45.511821848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:55:45.515383031Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
ts=2026-04-28T21:55:45.792966759Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40892"
level=info ts=2026-04-28T21:55:46.826883113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:55:46.831861487Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:55:48.097579581Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:48.197070675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:55:48.201771076Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:55:48.242774203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:55:48.258770744Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:55:48.700727905Z caller=checkpoint.go:372 msg="writing series" size="574 kB"
level=debug ts=2026-04-28T21:55:48.700755334Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-28T21:55:49.035264611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:55:49.044318139Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:55:50.634665206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:55:50.63884774Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:55:51.953878697Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T21:55:53.097897492Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:55:53.289565364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:55:53.29328692Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:55:55.206564753Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T21:55:57.197296176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:55:57.203705082Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:55:58.078950391Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:55:58.079377943Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:55:58.097177453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:55:58.097204224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:55:58.097212111Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:55:58.097208672Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:55:58.460146603Z caller=checkpoint.go:372 msg="writing series" size="156 kB"
level=info ts=2026-04-28T21:56:01.398197382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=23 total_comp="64 MB" avg_comp="2.8 MB" total_uncomp="530 MB" avg_uncomp="23 MB" full=22 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:56:01.405484711Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 23, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:56:01.713059001Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T21:56:02.900186679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="623 B" avg_comp="623 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:56:02.905154652Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=warn ts=2026-04-28T21:56:03.097776647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:04.966044181Z caller=checkpoint.go:372 msg="writing series" size="794 kB"
level=debug ts=2026-04-28T21:56:04.966071291Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-28T21:56:05.871059306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:56:05.879081002Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:56:08.097555834Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:08.219537906Z caller=checkpoint.go:372 msg="writing series" size="866 kB"
level=debug ts=2026-04-28T21:56:09.535345503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:56:11.046198201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:56:11.049495317Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:56:11.471815976Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=debug ts=2026-04-28T21:56:12.504891265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T21:56:12.506340693Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T21:56:12.858019124Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:56:13.097372082Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:14.724929743Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-04-28T21:56:14.73005525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:56:14.736346212Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:56:16.80325322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:56:16.811284334Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:56:17.978339644Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=debug ts=2026-04-28T21:56:17.978362891Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=4
level=warn ts=2026-04-28T21:56:18.097732163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:21.231164077Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=warn ts=2026-04-28T21:56:23.097991768Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:24.484005806Z caller=checkpoint.go:372 msg="writing series" size="488 kB"
level=debug ts=2026-04-28T21:56:27.738693465Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=warn ts=2026-04-28T21:56:28.098043151Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:56:28.098049204Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:56:28.098075266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:56:28.098084818Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T21:56:29.050331012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:56:29.059715578Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:56:30.990152841Z caller=checkpoint.go:372 msg="writing series" size="612 kB"
level=debug ts=2026-04-28T21:56:30.99017807Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-28T21:56:32.283235188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:56:32.287552444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T21:56:33.098051236Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:33.908287732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:56:34.157985191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:56:34.167388859Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:56:34.167415175Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:56:34.16744071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:56:34.173593694Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:56:34.242678203Z caller=checkpoint.go:372 msg="writing series" size="378 kB"
level=info ts=2026-04-28T21:56:35.822826381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:56:35.826447339Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:56:37.495932675Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-04-28T21:56:37.632252824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:56:37.637195328Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T21:56:38.098097271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:40.748546438Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-28T21:56:42.859571247Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:56:43.097920889Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:56:43.89096287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:56:43.896860991Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T21:56:44.002118987Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T21:56:44.043257661Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:56:44.047434963Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:56:44.047454508Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:56:44.04747408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:56:44.051531589Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:56:44.236304711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:56:44.241030127Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:56:45.098950069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:56:45.102893814Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:56:46.616745103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:56:46.625454816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:56:47.215332981Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:56:47.219016075Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:56:47.255555878Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=debug ts=2026-04-28T21:56:47.255585125Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T21:56:47.865918418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:56:47.873886226Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:56:48.098088787Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:56:48.485506033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:56:48.492053237Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:56:49.779403934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:56:49.782891418Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:56:50.508103526Z caller=checkpoint.go:372 msg="writing series" size="115 kB"
level=warn ts=2026-04-28T21:56:53.097806399Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:56:53.761100035Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T21:56:54.171525864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:56:54.17492968Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:56:55.154243211Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T21:56:56.078336386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:56:56.081972964Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:56:57.013727682Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-04-28T21:56:58.078964421Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:56:58.079915747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:56:58.097447975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:56:58.097468109Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:56:58.097477057Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:56:58.097571688Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:00.266550703Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-04-28T21:57:01.552385766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:57:01.552458988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="446 B" avg_comp="446 B" total_uncomp="36 kB" avg_uncomp="36 kB" idle=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T21:57:01.557268678Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-28T21:57:03.097921678Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:57:03.238260689Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:57:03.242957921Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:57:03.520354506Z caller=checkpoint.go:372 msg="writing series" size="73 kB"
level=info ts=2026-04-28T21:57:06.15436732Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:57:06.159473543Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T21:57:06.772502207Z caller=checkpoint.go:372 msg="writing series" size="95 kB"
level=warn ts=2026-04-28T21:57:08.097988025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:10.042762945Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T21:57:10.04278758Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=7
ts=2026-04-28T21:57:10.368072931Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:43902"
ts=2026-04-28T21:57:12.861212476Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T21:57:13.097888829Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:13.279187358Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T21:57:13.320226126Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:57:13.326470875Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:57:13.695112256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:57:13.703367193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T21:57:16.531723588Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-04-28T21:57:18.097722019Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:57:18.138788914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T21:57:18.143666737Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T21:57:18.3411326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:57:18.345115581Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T21:57:19.785109892Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T21:57:23.038292101Z caller=checkpoint.go:372 msg="writing series" size="19 kB"
level=warn ts=2026-04-28T21:57:23.097859126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:26.291376068Z caller=checkpoint.go:372 msg="writing series" size="170 kB"
level=info ts=2026-04-28T21:57:26.590103484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:57:26.599267138Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:57:27.702558335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:57:27.706145328Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:57:28.097155135Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:57:28.097181685Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:57:28.097189355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:57:28.097234901Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:57:28.112371149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:57:28.116047157Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T21:57:28.723579539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:57:28.732313638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:57:29.206318668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:57:29.210878446Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:57:29.543802349Z caller=checkpoint.go:372 msg="writing series" size="173 kB"
level=debug ts=2026-04-28T21:57:31.292683827Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:57:31.519946061Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:57:32.797012437Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:57:33.056690586Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:57:33.066907507Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T21:57:33.097495522Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:57:34.49660413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:57:34.500436828Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:57:35.059808534Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:57:35.066762219Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:57:36.049693488Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T21:57:37.29646316Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:57:37.302210232Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T21:57:38.097149802Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:39.303384427Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
ts=2026-04-28T21:57:40.369877657Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:52032"
level=info ts=2026-04-28T21:57:40.535828971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T21:57:40.539824837Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T21:57:41.294634148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:57:41.297943621Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T21:57:42.556065048Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-04-28T21:57:42.862787298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T21:57:42.966184165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:57:42.973424277Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:57:43.046167154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:57:43.04642792Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:57:43.05063326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:57:43.050643525Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T21:57:43.097367126Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:45.809397445Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T21:57:48.097679285Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:49.061828586Z caller=checkpoint.go:372 msg="writing series" size="255 kB"
level=debug ts=2026-04-28T21:57:49.061850269Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-04-28T21:57:52.314665315Z caller=checkpoint.go:372 msg="writing series" size="52 kB"
level=warn ts=2026-04-28T21:57:53.09782642Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:55.570927819Z caller=checkpoint.go:372 msg="writing series" size="2.1 MB"
level=debug ts=2026-04-28T21:57:55.570954545Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.2 MB" series=2
level=info ts=2026-04-28T21:57:58.079026542Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:57:58.081366217Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:57:58.098060671Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:57:58.098080686Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:57:58.098089961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:57:58.098092989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:57:58.846931382Z caller=checkpoint.go:372 msg="writing series" size="17 MB"
level=debug ts=2026-04-28T21:57:58.846961937Z caller=checkpoint.go:384 msg="flushing series" totalSize="17 MB" series=1
level=info ts=2026-04-28T21:57:59.184273929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:57:59.190149955Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:57:59.426763875Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:57:59.430356243Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T21:58:02.073720468Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=warn ts=2026-04-28T21:58:03.098240006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:58:04.909261311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:58:05.327514084Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-04-28T21:58:05.813818424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:58:05.820440627Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T21:58:08.097833724Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:58:08.58003558Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-04-28T21:58:08.753319325Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:58:08.764028609Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T21:58:10.371655413Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:58436"
level=info ts=2026-04-28T21:58:11.725434539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.7 MB" total_uncomp="72 MB" avg_uncomp="18 MB" full=3 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:58:11.733018717Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:58:11.833581621Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T21:58:12.865668158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:58:13.098228992Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:58:14.291818167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:44912"
level=debug ts=2026-04-28T21:58:15.087198915Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T21:58:15.087229897Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T21:58:17.786683157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:58:17.789902547Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:58:17.976680626Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T21:58:18.097508766Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:58:18.339266253Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-04-28T21:58:21.592511067Z caller=checkpoint.go:372 msg="writing series" size="233 kB"
level=info ts=2026-04-28T21:58:22.90150627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:58:22.905310801Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T21:58:23.097981753Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:24.245871071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:58:24.250480257Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T21:58:24.84536334Z caller=checkpoint.go:372 msg="writing series" size="176 kB"
level=info ts=2026-04-28T21:58:25.0894348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:58:25.093203775Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:58:25.692961575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:58:25.700942998Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:58:25.70096809Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:58:25.701001567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:58:25.708331011Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T21:58:28.098649538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:58:28.098678736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:58:28.098686919Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:58:28.098702597Z caller=checkpoint.go:384 msg="flushing series" totalSize="618 kB" series=3
level=warn ts=2026-04-28T21:58:28.098718727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:28.105716857Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000014.tmp new=/var/loki/wal/checkpoint.000014
level=info ts=2026-04-28T21:58:28.107645672Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010141835s
level=info ts=2026-04-28T21:58:29.192332317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:58:29.197369659Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:58:29.321777006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:58:29.329798485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T21:58:31.145276302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:58:31.1489839Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T21:58:33.097792163Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:35.045908409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:58:35.049663226Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:58:35.196271713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T21:58:35.199672132Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T21:58:35.372229598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:58:35.376527504Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T21:58:35.591262007Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T21:58:38.098129627Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T21:58:40.374475767Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:51530"
level=info ts=2026-04-28T21:58:41.096321408Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:58:41.104662921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:58:41.105296879Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:58:41.108235971Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:58:42.67303454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:58:42.68053542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T21:58:42.86766265Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:58:43.097742136Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T21:58:48.097353656Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:49.763674612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:58:49.769452819Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:58:52.62610305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:58:52.633949117Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T21:58:53.097283853Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:54.415870213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:58:54.41962836Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:58:55.341499774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:58:55.347120459Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:58:55.347142968Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:58:55.347167802Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:58:55.351804551Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T21:58:58.079191529Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:58:58.079668838Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:58:58.096905146Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T21:58:58.097066014Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-28T21:58:58.097077559Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T21:58:58.09709717Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=15.036µs success=true
level=warn ts=2026-04-28T21:58:58.097165394Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:58:58.097173999Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:58:58.097192509Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:58:58.097200758Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:58:58.097352643Z caller=checkpoint.go:372 msg="writing series" size="4.9 kB"
level=info ts=2026-04-28T21:59:00.93059905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:59:00.934204183Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:59:01.350787506Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T21:59:03.09733762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:04.723388532Z caller=checkpoint.go:372 msg="writing series" size="67 MB"
level=debug ts=2026-04-28T21:59:04.723452988Z caller=checkpoint.go:384 msg="flushing series" totalSize="67 MB" series=3
level=info ts=2026-04-28T21:59:07.333584149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:59:07.351095054Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T21:59:07.858296574Z caller=checkpoint.go:372 msg="writing series" size="729 kB"
level=warn ts=2026-04-28T21:59:08.097508684Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:08.634810548Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T21:59:08.634889246Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="438 B" avg_comp="438 B" total_uncomp="15 kB" avg_uncomp="15 kB" idle=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T21:59:08.648135389Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T21:59:11.040384454Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T21:59:11.059323997Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T21:59:11.117912082Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T21:59:11.117981929Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=2
ts=2026-04-28T21:59:12.870322146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T21:59:13.097817177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:13.536871394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:59:13.558670951Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:59:14.362909794Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-04-28T21:59:14.527666773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:59:14.538353318Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:59:16.476554951Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:59:16.491029405Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T21:59:17.541637398Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T21:59:17.616067746Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T21:59:18.097390366Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:19.522218034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:59:19.533011479Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:59:19.533044649Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:19.533101044Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:59:19.542607632Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T21:59:20.461465518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:59:20.474736777Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:59:20.474772002Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:20.474813166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:59:20.481740281Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T21:59:20.869604429Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T21:59:21.671199573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T21:59:21.678875117Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T21:59:23.097673332Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:23.729241624Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T21:59:23.732598795Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T21:59:24.12653577Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-28T21:59:24.126557927Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=4
level=info ts=2026-04-28T21:59:24.297511903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T21:59:24.300754914Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T21:59:24.345818616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T21:59:24.349843769Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T21:59:24.450000345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:59:24.453932845Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:59:24.453951884Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:24.453974351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:59:24.457158718Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T21:59:24.593741086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T21:59:24.601262572Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T21:59:27.375198291Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T21:59:28.098040377Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T21:59:28.098048413Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:28.09807389Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:59:28.09808429Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T21:59:30.628488581Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T21:59:33.098156505Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:33.482822993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T21:59:33.486323761Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T21:59:33.882324402Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T21:59:33.882345933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-28T21:59:34.094954255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T21:59:34.098550664Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T21:59:36.228472868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T21:59:37.133720582Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T21:59:38.097610267Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:39.737221451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T21:59:39.743376064Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T21:59:40.00816833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T21:59:40.011481481Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T21:59:40.146970779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T21:59:40.151195761Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T21:59:40.387265122Z caller=checkpoint.go:372 msg="writing series" size="221 kB"
level=info ts=2026-04-28T21:59:41.215245753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T21:59:41.220087898Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T21:59:42.298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T21:59:42.307208777Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T21:59:42.872087077Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=info ts=2026-04-28T21:59:42.907885819Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:59:42.911323375Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:59:42.911343621Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:42.911366239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:59:42.914131993Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T21:59:43.09743857Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:43.640092814Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-04-28T21:59:44.294194248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T21:59:44.301718107Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T21:59:46.893583449Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T21:59:48.097541251Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T21:59:48.354218183Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T21:59:48.358086185Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T21:59:50.148163539Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-28T21:59:50.148186657Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=warn ts=2026-04-28T21:59:53.097385416Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:53.399195229Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T21:59:56.619581083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T21:59:56.623305732Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T21:59:56.66886925Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-28T21:59:56.668910644Z caller=checkpoint.go:384 msg="flushing series" totalSize="15 MB" series=2
level=info ts=2026-04-28T21:59:56.871071114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:59:56.875164446Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:59:56.875187792Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:56.875210977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T21:59:56.878287458Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T21:59:57.343639391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:59:57.347503918Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T21:59:57.347523544Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T21:59:57.347546309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T21:59:57.351058359Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T21:59:57.537679139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T21:59:57.544545689Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T21:59:58.054114779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T21:59:58.057972047Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T21:59:58.079107726Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T21:59:58.079147877Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T21:59:58.097898906Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T21:59:58.097921802Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T21:59:58.097929633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T21:59:58.097922188Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T21:59:59.90596011Z caller=checkpoint.go:372 msg="writing series" size="920 kB"
level=info ts=2026-04-28T22:00:01.745481072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:00:01.749516034Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:00:02.471213066Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:00:02.474631569Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:00:03.097382895Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:00:03.158570535Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=debug ts=2026-04-28T22:00:03.794418766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:00:03.856441319Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=6 total_comp="14 MB" avg_comp="2.4 MB" total_uncomp="124 MB" avg_uncomp="21 MB" full=5 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:00:03.864506269Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:00:06.411776111Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T22:00:06.411798574Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-04-28T22:00:07.385374292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:00:07.39459029Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:00:08.063377144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:00:08.067720212Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T22:00:08.097489536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:08.38677659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:00:08.390307233Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:00:09.664830426Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=debug ts=2026-04-28T22:00:10.370214302Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:00:10.372241934Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:00:10.381942147Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:48392"
ts=2026-04-28T22:00:12.874302131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=debug ts=2026-04-28T22:00:12.917581562Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=warn ts=2026-04-28T22:00:13.097967243Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T22:00:14.299197734Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:40176"
level=info ts=2026-04-28T22:00:14.89760278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:00:14.903791052Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:00:16.12366963Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:00:16.127289778Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:00:16.127309636Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:00:16.127331165Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:00:16.131562244Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:00:16.170180914Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=info ts=2026-04-28T22:00:16.333439898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:00:16.33685629Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T22:00:18.097705523Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:18.565151973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:00:18.569598043Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:00:19.422724634Z caller=checkpoint.go:372 msg="writing series" size="7.4 kB"
level=debug ts=2026-04-28T22:00:22.67684163Z caller=checkpoint.go:372 msg="writing series" size="257 kB"
level=warn ts=2026-04-28T22:00:23.097801783Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:23.73365311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:00:23.738112433Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:00:25.686981849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:00:25.695421701Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:00:25.930067459Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T22:00:25.93008951Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.3 MB" series=6
level=info ts=2026-04-28T22:00:28.097923503Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:00:28.097923622Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:00:28.097948518Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:00:28.097955969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:00:29.182059499Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T22:00:31.154825999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:00:31.158499694Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:00:31.190659056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:00:31.194299837Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:00:31.314664822Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:00:31.319707978Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:00:31.85988774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:00:31.864595129Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-04-28T22:00:31.864629288Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:00:31.864652168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:00:31.868865624Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T22:00:32.435651825Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T22:00:33.097997414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:00:35.688538131Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-28T22:00:38.097229111Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:38.142467662Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:00:38.147069223Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:00:38.615883147Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:00:38.933505585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:00:38.939456128Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:00:38.941654568Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-04-28T22:00:40.383782127Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53518"
level=info ts=2026-04-28T22:00:41.988659245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:00:41.992769252Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:00:42.195056143Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T22:00:42.593744791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:00:42.597815284Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-28T22:00:42.875984874Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:00:43.097387423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:44.364771895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:00:44.368814113Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:00:45.03643001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:00:45.047152326Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:00:45.4474626Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-04-28T22:00:45.810715934Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46570"
level=warn ts=2026-04-28T22:00:48.097818232Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:00:48.700731393Z caller=checkpoint.go:372 msg="writing series" size="476 kB"
level=debug ts=2026-04-28T22:00:48.700755606Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-28T22:00:49.20482877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:00:49.209959363Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T22:00:50.549830274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:00:50.553137123Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:00:51.952794892Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T22:00:53.097529356Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:00:53.718142818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:00:53.724699925Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:00:55.206464684Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T22:00:56.149360636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="623 B" avg_comp="623 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:00:56.15289515Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:00:58.079042402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:00:58.079455108Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-04-28T22:00:58.080763685Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T22:00:58.08081156Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974903 n=1
level=debug ts=2026-04-28T22:00:58.080859463Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974903 n=1
level=info ts=2026-04-28T22:00:58.097155368Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:00:58.097173139Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:00:58.097181112Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:00:58.097211463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:00:58.459835177Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-04-28T22:00:59.327026558Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:00:59.331084419Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:01:01.71246294Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T22:01:01.78775684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:01:01.792423051Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T22:01:03.098122774Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:03.333727874Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:01:03.338339922Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:01:04.965033685Z caller=checkpoint.go:372 msg="writing series" size="540 kB"
level=info ts=2026-04-28T22:01:05.796112998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:01:05.805686519Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:01:08.057554691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:01:08.061011897Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T22:01:08.061029821Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:01:08.061051739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:01:08.064029593Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:01:08.097869551Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:01:08.218851943Z caller=checkpoint.go:372 msg="writing series" size="806 kB"
level=debug ts=2026-04-28T22:01:08.218878959Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-04-28T22:01:08.649186195Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T22:01:10.386269389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47878"
level=debug ts=2026-04-28T22:01:11.471089733Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-04-28T22:01:12.176605266Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:01:12.184391017Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:01:12.184414042Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:01:12.184439466Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:01:12.192727061Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T22:01:12.877575359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:01:13.097493663Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:13.365136186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:01:13.369030762Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:01:13.568255876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:01:14.306984489Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:01:14.310414689Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:01:14.724509412Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-28T22:01:17.978066091Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-04-28T22:01:18.097673902Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:01:21.230576392Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-04-28T22:01:22.245812264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:01:22.250760676Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:01:23.097997167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:23.125092718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:01:23.131830771Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T22:01:24.483979787Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-04-28T22:01:24.484001832Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=debug ts=2026-04-28T22:01:27.736336483Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T22:01:28.097466988Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:01:28.097517874Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:01:28.097526994Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:01:28.09753835Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:28.6091451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:01:28.615774688Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:01:29.36542253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:01:29.371164665Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:01:30.185443982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:01:30.193649521Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:01:30.990086831Z caller=checkpoint.go:372 msg="writing series" size="527 kB"
level=info ts=2026-04-28T22:01:31.076167177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:01:31.080291426Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:01:32.280052617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:01:32.284488616Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T22:01:33.097388254Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:01:34.243009152Z caller=checkpoint.go:372 msg="writing series" size="234 kB"
level=debug ts=2026-04-28T22:01:37.495386936Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-04-28T22:01:37.803265635Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:01:37.811770283Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:01:37.872059004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:01:37.875506298Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T22:01:38.097117109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T22:01:40.388850324Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:36968"
level=debug ts=2026-04-28T22:01:40.748058186Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T22:01:40.748081882Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T22:01:41.166622801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:01:41.170585315Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-28T22:01:42.879456767Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:01:43.098142939Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:43.449485702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:01:43.45734037Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:01:44.001069761Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T22:01:44.193822007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:01:44.197687901Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:01:44.961643691Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:01:45.620132259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:01:45.623968603Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:01:45.623986681Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:01:45.624006345Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:01:45.627250792Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:01:47.155595085Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:01:47.163569124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:01:47.163591517Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:01:47.163650801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:01:47.172804162Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:01:47.249727275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:01:47.252994272Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:01:47.254217152Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=warn ts=2026-04-28T22:01:48.098134674Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:48.592811864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:01:48.596826741Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:01:50.507578976Z caller=checkpoint.go:372 msg="writing series" size="116 kB"
level=info ts=2026-04-28T22:01:51.774569801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:01:51.778675567Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T22:01:53.097446205Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:01:53.760279193Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T22:01:54.954746267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:01:54.965258257Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:01:55.423738994Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:01:55.427255032Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:01:57.013353907Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=info ts=2026-04-28T22:01:58.013337001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:01:58.016987344Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:01:58.079196084Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:01:58.07960489Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:01:58.09709212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:01:58.097121203Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:01:58.097132114Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:01:58.097146089Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:01:58.488459511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:01:58.491916748Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:02:00.266789443Z caller=checkpoint.go:372 msg="writing series" size="169 kB"
level=warn ts=2026-04-28T22:02:03.098012496Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:03.519628586Z caller=checkpoint.go:372 msg="writing series" size="74 kB"
level=debug ts=2026-04-28T22:02:03.519651135Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-28T22:02:05.794588182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:02:05.802533145Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:02:06.192323887Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:02:06.773100287Z caller=checkpoint.go:372 msg="writing series" size="98 kB"
level=info ts=2026-04-28T22:02:06.826197575Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:02:06.831175505Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=warn ts=2026-04-28T22:02:08.097707519Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:10.041455801Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T22:02:10.041481436Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=2
ts=2026-04-28T22:02:10.391156877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:37714"
level=debug ts=2026-04-28T22:02:11.14121899Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T22:02:12.882041969Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T22:02:13.044464458Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:02:13.049256063Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T22:02:13.097701837Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:13.278716927Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:02:13.295721144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:02:13.300159887Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:02:14.032190614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:02:14.038816051Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:02:14.03883569Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:02:14.038861599Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:02:14.044705451Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:02:14.136114901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:02:14.139494849Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
ts=2026-04-28T22:02:15.815548905Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:46704"
level=debug ts=2026-04-28T22:02:16.531747112Z caller=checkpoint.go:372 msg="writing series" size="199 kB"
level=warn ts=2026-04-28T22:02:18.097660544Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:19.784873682Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T22:02:23.038053284Z caller=checkpoint.go:372 msg="writing series" size="20 kB"
level=warn ts=2026-04-28T22:02:23.097467461Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:02:23.272873644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:02:23.278727545Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:02:25.790204898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:02:25.799801239Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:02:26.070757845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:02:26.074861443Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:02:26.291017429Z caller=checkpoint.go:372 msg="writing series" size="217 kB"
level=info ts=2026-04-28T22:02:27.982865328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:02:27.990560358Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:02:28.097609795Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:02:28.097657801Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:02:28.09766602Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:02:28.097668303Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:02:28.704960359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:02:28.712220831Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:02:29.478304626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:02:29.482453851Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:02:29.54379808Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=info ts=2026-04-28T22:02:29.875733168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:02:29.879368142Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:02:32.797177029Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:02:32.936662076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:02:32.940798746Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:02:33.005024512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:02:33.008753337Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T22:02:33.097745138Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:02:34.021658885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:02:34.027760017Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:02:35.430850348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:02:35.43414628Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:02:36.050019161Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:02:37.224546691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:02:37.227655821Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:02:38.097537167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:39.302388013Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-04-28T22:02:41.172818853Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:02:41.17665099Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:02:42.555951556Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-04-28T22:02:42.88474544Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:02:43.097177575Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:43.366200931Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:02:45.808887864Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T22:02:48.097936606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:02:48.456460525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:02:48.465896076Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:02:49.061681987Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-04-28T22:02:49.061704281Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-04-28T22:02:51.322949206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:02:51.327076989Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:02:52.315284892Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T22:02:53.097315056Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:55.569388861Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-28T22:02:55.569412796Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=info ts=2026-04-28T22:02:57.863254779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:02:57.870760556Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:02:58.078952874Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:02:58.079285441Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:02:58.097638833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:02:58.097666352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:02:58.097676499Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:02:58.097748876Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:02:58.846455464Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-04-28T22:02:58.846483876Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=1
level=debug ts=2026-04-28T22:03:00.490143461Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:03:02.073956735Z caller=checkpoint.go:372 msg="writing series" size="215 kB"
level=info ts=2026-04-28T22:03:02.309268839Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:03:02.314324865Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T22:03:02.838733255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:03:02.844797267Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T22:03:03.097661727Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:03.482345009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:03:03.485947358Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:03:05.326635984Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-04-28T22:03:06.390508202Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:03:06.394001228Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:03:06.408069568Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:03:06.417009323Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T22:03:08.098166846Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:03:08.580148858Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
ts=2026-04-28T22:03:10.394727597Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53192"
level=info ts=2026-04-28T22:03:11.242852462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:03:11.246347722Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:03:11.832750526Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
ts=2026-04-28T22:03:12.88731742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:03:13.097180859Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:13.707221102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:03:13.711539493Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:03:14.489074146Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:03:15.086273296Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T22:03:15.086295106Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T22:03:17.770752008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:03:17.775164821Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:03:18.097587438Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:18.185843782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:03:18.190186294Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:03:18.33877699Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-04-28T22:03:20.180346608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:03:20.185464622Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:03:21.184423297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:03:21.192412259Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:03:21.591837824Z caller=checkpoint.go:372 msg="writing series" size="236 kB"
level=warn ts=2026-04-28T22:03:23.097703883Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:24.57757656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:03:24.582314469Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:03:24.582336149Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:03:24.582360742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:03:24.58599322Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:03:24.844891252Z caller=checkpoint.go:372 msg="writing series" size="179 kB"
level=info ts=2026-04-28T22:03:24.945062821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:03:24.948498077Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:03:25.245358198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:03:25.251913661Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:03:25.251933629Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:03:25.251962156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:03:25.25904817Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:03:28.097942176Z caller=checkpoint.go:384 msg="flushing series" totalSize="624 kB" series=3
level=info ts=2026-04-28T22:03:28.097976724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:03:28.097996717Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:03:28.098004387Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:03:28.098019899Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:28.106038636Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000015.tmp new=/var/loki/wal/checkpoint.000015
level=info ts=2026-04-28T22:03:28.108120909Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010868023s
level=info ts=2026-04-28T22:03:28.864610192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:03:28.869680583Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:03:33.09721157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:33.34481733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:03:33.348321097Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:03:34.656475714Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:03:34.660818905Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:03:36.297044982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:03:36.304195202Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T22:03:38.097669223Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:39.792834711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:03:39.797938138Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T22:03:42.888895555Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:03:43.097741036Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:43.503049774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:03:43.511725614Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:03:45.231204124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:03:45.238103091Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T22:03:48.097813006Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:03:48.927136524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:03:49.184023075Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:03:49.191478875Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:03:49.776202387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:03:49.779789436Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:03:50.037436079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:03:50.040963377Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:03:50.040986313Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:03:50.041008943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:03:50.043666623Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-28T22:03:53.09772146Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:03:53.192746443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:03:58.07898307Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:03:58.079212245Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:03:58.097459599Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:03:58.097491728Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:03:58.097482813Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T22:03:58.097500752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:03:58.097518793Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:03:58.09789554Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-28T22:03:58.097908907Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T22:03:58.09793505Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=19.379µs success=true
level=debug ts=2026-04-28T22:03:58.098256773Z caller=checkpoint.go:372 msg="writing series" size="5.2 kB"
level=info ts=2026-04-28T22:03:59.47099022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:03:59.477995497Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:04:01.351440924Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T22:04:01.917145686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:04:01.921876296Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T22:04:02.081437712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:04:02.084765621Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T22:04:03.097663466Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:04.721005719Z caller=checkpoint.go:372 msg="writing series" size="67 MB"
level=debug ts=2026-04-28T22:04:04.721039432Z caller=checkpoint.go:384 msg="flushing series" totalSize="67 MB" series=3
level=info ts=2026-04-28T22:04:05.176005786Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:04:05.193377807Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:04:05.754775237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:04:05.767740518Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:04:07.673926894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:04:07.689853642Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:04:07.860005732Z caller=checkpoint.go:372 msg="writing series" size="810 kB"
level=warn ts=2026-04-28T22:04:08.097416004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:04:08.341334394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:04:08.358238443Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:04:08.520930997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:04:08.52903468Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:04:10.176565166Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:04:10.187525769Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:04:11.122170902Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T22:04:11.122237371Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=2
ts=2026-04-28T22:04:12.89078588Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:04:13.097828942Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:04:13.517708379Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:04:13.528660199Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-28T22:04:14.317038048Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:56516"
level=debug ts=2026-04-28T22:04:14.363968987Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-04-28T22:04:14.766109139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:04:15.656181137Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:04:15.676065939Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:04:16.77138825Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:04:16.785684331Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:04:17.406974092Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:04:17.420491013Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:04:17.617672019Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T22:04:18.097431312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:20.870363282Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T22:04:22.766581278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:04:22.770699608Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:04:23.097642193Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:24.128258761Z caller=checkpoint.go:372 msg="writing series" size="4.7 MB"
level=debug ts=2026-04-28T22:04:24.128282658Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=4
level=debug ts=2026-04-28T22:04:27.375663013Z caller=checkpoint.go:372 msg="writing series" size="317 kB"
level=info ts=2026-04-28T22:04:28.097897731Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:04:28.097907433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:28.097925673Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:04:28.097934404Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:04:29.437035765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:04:29.443469131Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:04:30.628794326Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T22:04:33.097897777Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:33.882849103Z caller=checkpoint.go:372 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-28T22:04:33.88287083Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-28T22:04:35.052220723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:04:35.058456539Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:04:37.134450625Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T22:04:38.097481872Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:04:38.127574944Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:04:38.135992645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:04:40.388558368Z caller=checkpoint.go:372 msg="writing series" size="271 kB"
level=info ts=2026-04-28T22:04:41.17587013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:04:41.18450207Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:04:41.184527378Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:04:41.184555543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:04:41.192134038Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:04:42.496388033Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:04:42.49804503Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T22:04:42.895386384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=info ts=2026-04-28T22:04:42.956252391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:04:42.959779124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T22:04:43.0979374Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:43.640946295Z caller=checkpoint.go:372 msg="writing series" size="30 kB"
level=info ts=2026-04-28T22:04:43.696095238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:04:43.700359629Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:04:46.273561034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:04:46.277633129Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:04:46.893465355Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T22:04:48.097224956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:04:49.376570861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="759 B" avg_comp="759 B" total_uncomp="25 kB" avg_uncomp="25 kB" idle=1 labels="{stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:04:49.376639096Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-28T22:04:49.380207823Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:04:50.149637696Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-28T22:04:50.149665286Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=warn ts=2026-04-28T22:04:53.097439585Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:04:53.399651228Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T22:04:53.410660479Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:04:53.416094174Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T22:04:53.692840465Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:04:53.698134262Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:04:53.947753659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:04:53.951345326Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:04:54.336655946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:04:54.344720943Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:04:54.622521338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:04:54.625972598Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:04:55.518865242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:04:55.523652625Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:04:55.870636455Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:04:55.87879104Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:04:56.671710093Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-28T22:04:56.67173485Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-04-28T22:04:58.078963315Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:04:58.079462594Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T22:04:58.097208687Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:04:58.097228216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:04:58.09726274Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:04:58.097273254Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:04:58.769019931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:04:58.772504687Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:04:59.906174009Z caller=checkpoint.go:372 msg="writing series" size="830 kB"
level=warn ts=2026-04-28T22:05:03.097988933Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:03.158791091Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=info ts=2026-04-28T22:05:03.381392948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:05:03.389385149Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:05:03.512049263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:05:03.515683406Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:05:03.909077095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:05:03.912853255Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:05:06.41278702Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T22:05:06.41280929Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-28T22:05:07.816029771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:05:07.822392641Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T22:05:08.097398964Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:05:08.839537934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:05:08.843053207Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:05:09.665664824Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
ts=2026-04-28T22:05:12.897430579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=debug ts=2026-04-28T22:05:12.917815578Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=warn ts=2026-04-28T22:05:13.097714235Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:16.171126991Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=warn ts=2026-04-28T22:05:18.097530016Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:05:18.373279101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:05:18.378545817Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:05:19.423914844Z caller=checkpoint.go:372 msg="writing series" size="8.0 kB"
level=info ts=2026-04-28T22:05:20.177988362Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:05:20.18694841Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:05:22.676674215Z caller=checkpoint.go:372 msg="writing series" size="46 kB"
level=warn ts=2026-04-28T22:05:23.097611107Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:24.320116428Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:05:25.356799685Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:05:25.931600887Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T22:05:25.931648211Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=6
level=info ts=2026-04-28T22:05:26.270481567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:05:26.273794279Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:05:26.675130982Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:05:26.681267002Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:05:27.980007368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:05:27.9881931Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:05:28.097091222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:05:28.097120639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:05:28.097130752Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:05:28.097154474Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:29.183171732Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-28T22:05:32.43587418Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T22:05:33.098115032Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:05:33.449408307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:05:33.453839963Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:05:35.148294288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:05:35.152044891Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:05:35.688869273Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T22:05:37.391438864Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:05:37.398250144Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T22:05:38.097465617Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:38.942332788Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
ts=2026-04-28T22:05:40.402481558Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47222"
level=info ts=2026-04-28T22:05:41.138106112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:05:41.147204709Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:05:41.729261001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:05:41.733213279Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:05:41.733230112Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:05:41.733250477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:05:41.736053219Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:05:42.196101738Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T22:05:42.900245584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:05:43.097332786Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:05:44.477972555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:05:44.481441884Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:05:45.448198653Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T22:05:46.515469331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:05:46.521057539Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:05:46.521076054Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:05:46.521098012Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:05:46.525469323Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T22:05:48.097893375Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:05:48.446096967Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:05:48.45102782Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=debug ts=2026-04-28T22:05:48.701345394Z caller=checkpoint.go:372 msg="writing series" size="611 kB"
level=debug ts=2026-04-28T22:05:48.701367158Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=debug ts=2026-04-28T22:05:50.66517622Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:05:51.000256569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:05:51.00454069Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:05:51.056647313Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:05:51.059928296Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:05:51.528836597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:05:51.532891558Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:05:51.9541319Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T22:05:53.097951437Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:55.207297244Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T22:05:55.895736043Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:05:55.899161932Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:05:55.899182125Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:05:55.89920251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:05:55.90197958Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:05:56.805426582Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:05:56.809705087Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:05:56.809726618Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:05:56.809753713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:05:56.813059588Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:05:57.794921797Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:05:57.803772741Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:05:58.07904817Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:05:58.079078907Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:05:58.097394009Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:05:58.097423045Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:05:58.097433137Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:05:58.09750823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:05:58.460271701Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-04-28T22:05:58.798856099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:05:58.802574619Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:06:01.713427555Z caller=checkpoint.go:372 msg="writing series" size="63 kB"
level=info ts=2026-04-28T22:06:01.830670734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:06:01.836168361Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:06:01.908547031Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="623 B" avg_comp="623 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:06:01.911970001Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:06:02.907436648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:06:02.910928357Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:06:02.910951835Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:06:02.910973452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:06:02.913715308Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:06:03.097631536Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:04.966384585Z caller=checkpoint.go:372 msg="writing series" size="536 kB"
level=warn ts=2026-04-28T22:06:08.097359796Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:08.219088148Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-04-28T22:06:08.219110702Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=debug ts=2026-04-28T22:06:08.332950209Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:06:08.898553751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:06:08.90206299Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:06:09.156650561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:06:09.160458486Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:06:11.472571769Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-04-28T22:06:11.918421298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:06:11.928046443Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T22:06:12.901713941Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:06:13.098048986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:14.725163521Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
ts=2026-04-28T22:06:15.82745681Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39492"
level=debug ts=2026-04-28T22:06:17.978381233Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-04-28T22:06:18.098004186Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:20.098029812Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:06:21.231119356Z caller=checkpoint.go:372 msg="writing series" size="252 kB"
level=info ts=2026-04-28T22:06:21.332301055Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:06:21.338113266Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:06:22.041145113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:06:22.045458984Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T22:06:22.045481666Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:06:22.045506564Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:06:22.049016324Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:06:22.699737613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:06:22.703129134Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:06:23.098141345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:24.484855534Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-04-28T22:06:24.484878636Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T22:06:24.716342403Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:06:24.722291329Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:06:26.732173855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:06:26.740268834Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:06:27.737655166Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T22:06:28.097791296Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:06:28.097816871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T22:06:28.097811211Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:06:28.097826018Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:06:30.497175984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:06:30.500459579Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:06:30.500478096Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:06:30.500501034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:06:30.503270187Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:06:30.990879018Z caller=checkpoint.go:372 msg="writing series" size="643 kB"
level=info ts=2026-04-28T22:06:32.606668691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:06:32.612878943Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=warn ts=2026-04-28T22:06:33.097586407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:06:33.953566242Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:06:33.957394567Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:06:33.957414678Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:06:33.957435991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:06:33.960015089Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:06:34.24318362Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=debug ts=2026-04-28T22:06:34.243205208Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-04-28T22:06:34.765412693Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:06:34.768822114Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:06:35.353462533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:06:35.356784245Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:06:36.310857518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:06:36.314510754Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:06:36.883110988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:06:36.88737026Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:06:37.496217739Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-04-28T22:06:38.097210026Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:06:39.005798109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:06:39.00958352Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:06:39.511181811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:06:39.519190558Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:06:40.093098467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:06:40.097759977Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:06:40.323361971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:06:40.328294218Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:06:40.749072161Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-28T22:06:42.903272323Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:06:43.098139067Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:44.001996766Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T22:06:44.740026195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:06:44.746879666Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=info ts=2026-04-28T22:06:45.133290431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:06:45.137506092Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:06:45.184665779Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:06:45.18822518Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:06:45.188246863Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:06:45.188268093Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:06:45.191237538Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:06:45.423043311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T22:06:45.829872267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39808"
level=debug ts=2026-04-28T22:06:47.256177175Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T22:06:47.821570569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:06:47.825296471Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:06:48.097458635Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:06:49.060730314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:06:49.064934414Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:06:50.508735652Z caller=checkpoint.go:372 msg="writing series" size="119 kB"
level=info ts=2026-04-28T22:06:51.172821553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:06:51.176584561Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T22:06:53.097831503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:06:53.761585697Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-04-28T22:06:54.464297247Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:06:55.320822087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:06:55.329335754Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:06:57.014324002Z caller=checkpoint.go:372 msg="writing series" size="175 kB"
level=debug ts=2026-04-28T22:06:57.014345776Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-28T22:06:58.078960236Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:06:58.079407431Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:06:58.097562589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:06:58.097593029Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T22:06:58.097589339Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:06:58.097602962Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:06:59.970830704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:06:59.974554401Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:07:00.267650582Z caller=checkpoint.go:372 msg="writing series" size="178 kB"
level=info ts=2026-04-28T22:07:00.629672006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:07:00.632914053Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=warn ts=2026-04-28T22:07:03.09787115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:07:03.520416822Z caller=checkpoint.go:372 msg="writing series" size="76 kB"
level=info ts=2026-04-28T22:07:06.648664543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:07:06.65230887Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:07:06.773880594Z caller=checkpoint.go:372 msg="writing series" size="103 kB"
level=warn ts=2026-04-28T22:07:08.098053159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:09.617986523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:07:09.621606749Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:07:10.043031066Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T22:07:10.043057488Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
ts=2026-04-28T22:07:12.905667432Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:07:13.098233584Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:07:13.279805436Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:07:13.574656198Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:07:13.578812029Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:07:14.260850952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:07:14.268123576Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:07:15.197101506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:07:15.207928139Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:07:15.353334642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:07:15.358480135Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:07:16.532947588Z caller=checkpoint.go:372 msg="writing series" size="200 kB"
level=warn ts=2026-04-28T22:07:18.098067311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:07:19.341718417Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:07:19.341791356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="175 kB" avg_comp="175 kB" total_uncomp="889 kB" avg_uncomp="889 kB" idle=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:07:19.346195304Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:07:19.785433523Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T22:07:20.757311417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:07:20.762120696Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:07:20.819381988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:07:20.822759049Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T22:07:20.82277749Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:07:20.822798167Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:07:20.825432842Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:07:23.038254101Z caller=checkpoint.go:372 msg="writing series" size="21 kB"
level=warn ts=2026-04-28T22:07:23.097588989Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:24.667894035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:07:24.675067993Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:07:25.072042073Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:07:25.07549096Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:07:26.292212091Z caller=checkpoint.go:372 msg="writing series" size="272 kB"
level=info ts=2026-04-28T22:07:27.119873326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:07:27.123496921Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:07:27.527528746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=03812195df7ecfc6 immediate=false num_chunks=1 total_comp="912 kB" avg_comp="912 kB" total_uncomp="7.2 MB" avg_uncomp="7.2 MB" synced=1 labels="{stream=\"integration-service\"}"
level=error ts=2026-04-28T22:07:27.532559312Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3812195df7ecfc6/19dd5d17417%3A19dd601bfc4%3Accfa9339?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration-service\"}"
level=error ts=2026-04-28T22:07:27.532583378Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:07:27.532607805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:07:27.536839804Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:07:28.097706323Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:07:28.097734543Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:07:28.097742872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:07:28.097766914Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:07:29.544564067Z caller=checkpoint.go:372 msg="writing series" size="219 kB"
level=info ts=2026-04-28T22:07:32.70915514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:07:32.713072739Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:07:32.798417943Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:07:32.813487153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:07:32.816898284Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:07:33.097881762Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:33.729964061Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:07:33.734198188Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:07:36.050921045Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:07:37.824758764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:07:37.833300668Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T22:07:38.097568844Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:38.102652583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:07:38.108251793Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:07:38.280905659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:07:38.290014534Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:07:38.36066368Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:07:38.364157714Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:07:39.303497881Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=debug ts=2026-04-28T22:07:42.556558036Z caller=checkpoint.go:372 msg="writing series" size="35 kB"
ts=2026-04-28T22:07:42.908209274Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:07:43.098035621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:44.021213531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:07:44.026722461Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:07:44.370833705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:07:44.374811439Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:07:45.809763282Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T22:07:48.097207129Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:48.294988382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:07:48.301310014Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:07:49.062714052Z caller=checkpoint.go:372 msg="writing series" size="256 kB"
level=debug ts=2026-04-28T22:07:49.06273682Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=12
level=debug ts=2026-04-28T22:07:50.193276785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:07:52.316085835Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=info ts=2026-04-28T22:07:52.499987249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:07:52.503781949Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:07:52.691603749Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T22:07:53.097953553Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:07:55.571339461Z caller=checkpoint.go:372 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-28T22:07:55.571360264Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.4 MB" series=2
level=info ts=2026-04-28T22:07:56.437650212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:07:56.44240039Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:07:56.60479717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:07:56.608347828Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=info ts=2026-04-28T22:07:58.079155295Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:07:58.079190415Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:07:58.097567929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:07:58.097591937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:07:58.09759969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:07:58.097596433Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:07:58.701848298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:07:58.705286992Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:07:58.842764594Z caller=checkpoint.go:372 msg="writing series" size="18 MB"
level=debug ts=2026-04-28T22:07:58.842796317Z caller=checkpoint.go:384 msg="flushing series" totalSize="18 MB" series=1
level=info ts=2026-04-28T22:07:59.664408576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:07:59.667913747Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:08:00.152415804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:08:00.155802295Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:08:02.074705162Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=warn ts=2026-04-28T22:08:03.097376714Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:08:05.327484871Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=warn ts=2026-04-28T22:08:08.09796545Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:08.482506861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:08:08.486450444Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:08:08.580916927Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-04-28T22:08:09.641409879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:08:09.646243526Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:08:09.853892952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:08:09.860987339Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:08:10.267319189Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:08:10.273539443Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:08:10.385800831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:08:10.396814284Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:08:10.396841483Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:08:10.39686663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:08:10.405756193Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:08:11.833508255Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T22:08:12.580857867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:08:12.585083915Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
ts=2026-04-28T22:08:12.910741625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:08:13.097789782Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:08:15.087716761Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T22:08:15.087739209Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T22:08:15.81000573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:08:15.813516881Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:08:18.097430072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:08:18.339324097Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=debug ts=2026-04-28T22:08:18.519036251Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:08:20.454870324Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:08:20.461800211Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:08:21.021553358Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:08:21.025345631Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:08:21.029383228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:08:21.037037161Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:08:21.593399486Z caller=checkpoint.go:372 msg="writing series" size="243 kB"
level=warn ts=2026-04-28T22:08:23.098113897Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:23.394994221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:08:23.402320219Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:08:24.753421143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:08:24.757856971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:08:24.846302272Z caller=checkpoint.go:372 msg="writing series" size="182 kB"
level=warn ts=2026-04-28T22:08:28.097162039Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:28.097177406Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:08:28.097198863Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:08:28.097207556Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:08:28.098248605Z caller=checkpoint.go:384 msg="flushing series" totalSize="634 kB" series=3
level=info ts=2026-04-28T22:08:28.104938784Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000016.tmp new=/var/loki/wal/checkpoint.000016
level=info ts=2026-04-28T22:08:28.106770231Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.008637492s
level=debug ts=2026-04-28T22:08:30.859014044Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:08:30.860592813Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T22:08:33.097669886Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:34.001031224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:08:34.006607094Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:08:35.263265562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:08:35.273360319Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:08:35.273389388Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:08:35.273458386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:08:35.280738546Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:08:36.633376824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:08:36.636893983Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:08:38.097979625Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:39.86277931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:08:39.867183308Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=info ts=2026-04-28T22:08:40.978838256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:08:40.982641591Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:08:41.897994184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:08:41.901396852Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:08:42.16908761Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:08:42.172567825Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-28T22:08:42.9132939Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:08:43.09736177Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:45.916301305Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:08:45.920052862Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:08:46.868737597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:08:46.876336411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T22:08:48.098053589Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:49.480138329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:08:49.484331564Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:08:49.484350502Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:08:49.484371463Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:08:49.487011882Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:08:50.965489446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:08:50.969096163Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:08:52.519490755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="220 kB" avg_comp="220 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" idle=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:08:52.519540592Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-28T22:08:52.526839413Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:08:52.72860269Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:08:52.735541495Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:08:52.819763901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="325 B" avg_comp="325 B" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" idle=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:08:52.823716047Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-28T22:08:53.097248799Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:08:55.976245071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:08:55.979756095Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:08:58.078991668Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:08:58.078999176Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T22:08:58.096762482Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T22:08:58.096972011Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000017
level=info ts=2026-04-28T22:08:58.096984871Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T22:08:58.097004659Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=14.963µs success=true
level=info ts=2026-04-28T22:08:58.097172601Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:08:58.097193974Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:08:58.097201856Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:08:58.097231526Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:08:58.097269147Z caller=checkpoint.go:372 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-28T22:09:01.350840033Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T22:09:03.09768109Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:03.427320318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:09:03.434555222Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:09:03.439556348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:09:03.443075087Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:09:03.555456519Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:09:03.559473032Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:09:04.711020658Z caller=checkpoint.go:372 msg="writing series" size="67 MB"
level=debug ts=2026-04-28T22:09:04.71105439Z caller=checkpoint.go:384 msg="flushing series" totalSize="67 MB" series=3
level=info ts=2026-04-28T22:09:05.363839112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:09:05.380193799Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:09:05.38205398Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:09:05.382100033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:09:05.399938716Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:09:07.864032118Z caller=checkpoint.go:372 msg="writing series" size="674 kB"
level=warn ts=2026-04-28T22:09:08.097367021Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:08.126106038Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:09:08.13981985Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:09:09.689470552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:09:09.699989699Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:09:10.968702394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:09:10.979886841Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:09:11.015888427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:09:11.026646431Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:09:11.115389297Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T22:09:11.117073783Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.4 MB" series=2
level=info ts=2026-04-28T22:09:11.620977156Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:09:11.632387044Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
ts=2026-04-28T22:09:12.915693311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:09:13.097806464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:13.268529678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:09:13.281398617Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:09:14.363187711Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=debug ts=2026-04-28T22:09:17.616795326Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=info ts=2026-04-28T22:09:17.961909765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:09:17.980655053Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T22:09:18.097414108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:18.22281552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:09:18.235295878Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:09:18.851870434Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:09:18.893208327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:09:18.903723179Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:09:20.027969977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:09:20.048440382Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:09:20.869803764Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T22:09:21.107926432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:09:21.114780817Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-28T22:09:23.097966175Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:24.126864377Z caller=checkpoint.go:372 msg="writing series" size="4.5 MB"
level=debug ts=2026-04-28T22:09:24.126885927Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=4
level=info ts=2026-04-28T22:09:25.023583835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:09:25.02975822Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:09:26.54204598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:09:26.783128107Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:09:26.78632447Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:09:27.375426335Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-04-28T22:09:28.097736709Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:09:28.097746113Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:28.097767936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:09:28.097776653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:09:28.795538765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:09:28.80376202Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:09:29.646585849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:09:29.650075576Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:09:30.628130631Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T22:09:30.632147488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:09:30.635782369Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=warn ts=2026-04-28T22:09:33.097677054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:33.881672639Z caller=checkpoint.go:372 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-28T22:09:33.88169501Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=info ts=2026-04-28T22:09:35.874940659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:09:35.881146668Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:09:37.133809754Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T22:09:38.097131882Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:39.666743105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:09:39.67001329Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:09:40.387445565Z caller=checkpoint.go:372 msg="writing series" size="365 kB"
level=info ts=2026-04-28T22:09:42.13213576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:09:42.136108727Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-28T22:09:42.919244082Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=info ts=2026-04-28T22:09:43.007137518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:09:43.010568591Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T22:09:43.09749653Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:09:43.29113748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:09:43.296516731Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:09:43.639900452Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
ts=2026-04-28T22:09:44.339854294Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43774"
level=info ts=2026-04-28T22:09:45.524581393Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:09:45.528121443Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:09:45.654507598Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:09:45.658648303Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:09:45.704978206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:09:45.711125734Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:09:45.7111464Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:09:45.711173498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:09:45.717315657Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-28T22:09:45.838693062Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52312"
level=debug ts=2026-04-28T22:09:46.892610205Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T22:09:47.850409685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:09:47.85474121Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T22:09:48.097173054Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:50.148264385Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-28T22:09:50.148298502Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=warn ts=2026-04-28T22:09:53.097665244Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:53.398874865Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T22:09:53.461187244Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:09:53.461230377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="141 B" avg_comp="141 B" total_uncomp="34 kB" avg_uncomp="34 kB" idle=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:09:53.465356931Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:09:53.644671072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:09:53.649581416Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:09:53.649601053Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:09:53.761663143Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="122 B" avg_comp="122 B" total_uncomp="60 kB" avg_uncomp="60 kB" idle=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:09:53.766136646Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:09:54.062049644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:09:54.066663515Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:09:56.669744477Z caller=checkpoint.go:372 msg="writing series" size="15 MB"
level=debug ts=2026-04-28T22:09:56.669768049Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-04-28T22:09:57.925792569Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:09:57.92937491Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:09:58.078993616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:09:58.079741389Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:09:58.097651025Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:09:58.097677694Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:09:58.097686358Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:09:58.097711641Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:09:58.779811884Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:09:59.905970981Z caller=checkpoint.go:372 msg="writing series" size="841 kB"
level=info ts=2026-04-28T22:10:00.598640083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:10:00.605534685Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:10:00.747019095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:10:00.750454485Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:10:02.886761194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:10:02.894291093Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T22:10:03.097930803Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:03.158105243Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=info ts=2026-04-28T22:10:03.554017561Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:10:03.560284129Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:10:05.301956959Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:10:05.305933301Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:10:05.883557186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:10:05.888224251Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:10:05.950298218Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:10:05.954857192Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:10:06.411996042Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T22:10:06.41201847Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-28T22:10:07.654935554Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:10:07.660092267Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:10:07.76247894Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:10:07.767058484Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:10:07.767078157Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:10:07.76709882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:10:07.769901415Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-28T22:10:08.098231637Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:08.517662955Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:10:08.521144927Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:10:08.608369917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:10:08.616505713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:10:09.664707408Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-04-28T22:10:09.856087294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:10:09.864360301Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T22:10:10.416360822Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:59672"
level=info ts=2026-04-28T22:10:11.909734432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:10:11.913909771Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:10:12.917218254Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
ts=2026-04-28T22:10:12.920394537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:10:13.097862784Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:14.75305132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:10:14.75707084Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:10:15.112389488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:10:15.116204254Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
ts=2026-04-28T22:10:15.840205389Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54560"
level=debug ts=2026-04-28T22:10:16.170739285Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=warn ts=2026-04-28T22:10:18.09718732Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:19.423147164Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-04-28T22:10:22.246585625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:10:22.256837426Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:10:22.676299814Z caller=checkpoint.go:372 msg="writing series" size="62 kB"
level=warn ts=2026-04-28T22:10:23.097792986Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:23.193259536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:10:23.196952375Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:10:24.317736666Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:10:24.321412478Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:10:24.452812783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:10:25.929733369Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T22:10:25.929754645Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=6
level=info ts=2026-04-28T22:10:27.067890457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:10:27.073983534Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=warn ts=2026-04-28T22:10:28.097733489Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:28.097741619Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:10:28.097766454Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:10:28.097776942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:10:28.624543756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:10:28.628153928Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:10:29.121024015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:10:29.124487888Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-28T22:10:29.181768908Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T22:10:31.000665721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:10:31.005642988Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:10:31.438854371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:10:31.44307031Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:10:31.808826219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:10:31.813099095Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:10:32.435515373Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T22:10:33.097875794Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:33.947751627Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:10:33.951727795Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:10:35.352736647Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:10:35.356147906Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:10:35.688265831Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=warn ts=2026-04-28T22:10:38.097153946Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:38.941009399Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T22:10:39.251871106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:10:39.257071056Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:10:42.194969482Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T22:10:42.220255817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:10:42.224526427Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T22:10:42.922186109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:10:43.097953426Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:45.447431437Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
ts=2026-04-28T22:10:45.841757653Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:32978"
level=info ts=2026-04-28T22:10:46.057864638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:10:46.062071009Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-28T22:10:48.097717578Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:48.700484955Z caller=checkpoint.go:372 msg="writing series" size="461 kB"
level=debug ts=2026-04-28T22:10:48.700504192Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=7
level=info ts=2026-04-28T22:10:49.055224888Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:10:49.058896341Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:10:49.392132703Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:10:49.396474923Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:10:49.501721294Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:10:51.21085541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:10:51.214527675Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:10:51.246607881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:10:51.253581764Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:10:51.505738438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:10:51.509432819Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:10:51.54942517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:10:51.55616015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:10:51.556174585Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:10:51.556197467Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:10:51.563574837Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:10:51.953515869Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T22:10:53.097265098Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:10:53.70880687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:10:53.718374019Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:10:55.205849022Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T22:10:58.080680804Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:10:58.081041409Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:10:58.097325967Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:10:58.097341345Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:10:58.09735453Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:10:58.097364297Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:10:58.415650953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:10:58.421118742Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:10:58.459256458Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=debug ts=2026-04-28T22:10:59.482689373Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:11:01.712028828Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T22:11:03.097213312Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:04.180137655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=7 total_comp="17 MB" avg_comp="2.4 MB" total_uncomp="147 MB" avg_uncomp="21 MB" full=6 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:11:04.191077554Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:11:04.875141457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:11:04.880306425Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:11:04.966020787Z caller=checkpoint.go:372 msg="writing series" size="567 kB"
level=info ts=2026-04-28T22:11:05.618639109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:11:05.621981391Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:11:05.73134621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:11:05.738385808Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T22:11:08.097662174Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:08.178844348Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:11:08.182581592Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:11:08.219116383Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-04-28T22:11:08.219132225Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-04-28T22:11:08.379012128Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:11:08.382790533Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=info ts=2026-04-28T22:11:08.961530424Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="623 B" avg_comp="623 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:11:08.965019254Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:11:09.333203511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:11:09.339554743Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:11:11.146735986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:11:11.150390114Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:11:11.471372389Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-04-28T22:11:12.923808778Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:11:13.097396127Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:14.724189948Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-04-28T22:11:14.927843672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:11:14.931586198Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:11:15.056267378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:11:15.060488161Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T22:11:15.843829153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51796"
level=info ts=2026-04-28T22:11:16.026546628Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:11:16.03068294Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:11:17.97816583Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-04-28T22:11:18.097876612Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:21.230012516Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-04-28T22:11:23.097558005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:24.483372266Z caller=checkpoint.go:372 msg="writing series" size="519 kB"
level=debug ts=2026-04-28T22:11:24.483394251Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T22:11:25.34419361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:11:25.349350435Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-28T22:11:27.736832985Z caller=checkpoint.go:372 msg="writing series" size="211 kB"
level=info ts=2026-04-28T22:11:28.097728195Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:11:28.097759521Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:11:28.097768044Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:11:28.097764073Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:28.735751588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:11:28.745087166Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:11:29.09232029Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:11:29.095777841Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:11:30.897800426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:11:30.901923343Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:11:30.989428361Z caller=checkpoint.go:372 msg="writing series" size="498 kB"
level=info ts=2026-04-28T22:11:32.607425367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:11:32.611441754Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-28T22:11:33.097379689Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:33.509828168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:11:33.513806046Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:11:33.660848186Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:11:33.744916713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:11:33.748209022Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:11:34.242982375Z caller=checkpoint.go:372 msg="writing series" size="360 kB"
level=debug ts=2026-04-28T22:11:34.24300429Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-04-28T22:11:34.365452691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:11:34.373153791Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:11:35.492671492Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:11:36.242746596Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:11:36.248514069Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T22:11:36.83711171Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:11:36.840565918Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:11:37.495492394Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-04-28T22:11:38.097379615Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:39.860457395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:11:39.865731229Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:11:40.748410233Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
ts=2026-04-28T22:11:42.92567269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:11:43.097728004Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:44.001311336Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T22:11:44.108860615Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:11:44.112341988Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:11:44.138410157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:11:44.141771413Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:11:47.255244837Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=info ts=2026-04-28T22:11:48.057244321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:11:48.06068449Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:11:48.060705819Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:11:48.060728328Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:11:48.063483901Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=warn ts=2026-04-28T22:11:48.098054909Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:11:49.469235106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:11:49.472572284Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:11:50.507698168Z caller=checkpoint.go:372 msg="writing series" size="123 kB"
level=info ts=2026-04-28T22:11:51.074278672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:11:51.082577471Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:11:51.086951022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:11:51.093600259Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:11:51.666820261Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:11:51.671496539Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-28T22:11:53.097676912Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:53.760737834Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T22:11:54.867733442Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:11:54.871104219Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:11:57.013661602Z caller=checkpoint.go:372 msg="writing series" size="180 kB"
level=debug ts=2026-04-28T22:11:57.013682197Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-28T22:11:58.079215144Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:11:58.079689015Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:11:58.097567049Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:11:58.097577547Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:11:58.097593056Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:11:58.097601716Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:11:59.348820545Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:12:00.267152036Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:12:02.956848804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:12:02.962288202Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=warn ts=2026-04-28T22:12:03.097708323Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:03.519647533Z caller=checkpoint.go:372 msg="writing series" size="77 kB"
level=info ts=2026-04-28T22:12:03.740208877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:12:03.743836718Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:12:03.810044241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:12:03.81428864Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:12:05.362758572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:12:05.366985077Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:12:05.908761054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:12:05.912521432Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:12:06.772901008Z caller=checkpoint.go:372 msg="writing series" size="109 kB"
level=warn ts=2026-04-28T22:12:08.098037743Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:10.041445793Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T22:12:10.041472401Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
ts=2026-04-28T22:12:10.421984461Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:44138"
ts=2026-04-28T22:12:12.928010373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:12:13.098025Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:13.279001107Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:12:14.06511069Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:12:14.070968509Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
ts=2026-04-28T22:12:14.348965885Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48788"
level=info ts=2026-04-28T22:12:14.732666851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:12:14.73681156Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:12:15.458216807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:12:15.462307059Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:12:15.96875105Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:12:15.977033003Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:12:16.531642983Z caller=checkpoint.go:372 msg="writing series" size="201 kB"
level=warn ts=2026-04-28T22:12:18.097996084Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:12:18.480501964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:12:18.484067554Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:12:19.690527891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:12:19.695004913Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:12:19.784314819Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T22:12:20.769261477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:12:20.775981865Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:12:23.037723321Z caller=checkpoint.go:372 msg="writing series" size="22 kB"
level=warn ts=2026-04-28T22:12:23.098128052Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:23.142120787Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:12:23.143691326Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T22:12:24.16707461Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:12:24.17041738Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:12:25.525209611Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:12:25.533002132Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:12:26.290883135Z caller=checkpoint.go:372 msg="writing series" size="328 kB"
level=info ts=2026-04-28T22:12:26.814111651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:12:26.81853353Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:12:26.818564407Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:26.81858697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:12:26.821324888Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:12:26.901325341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:12:26.904778355Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:12:26.904804144Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:27.014039124Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:12:27.019211474Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:12:27.019230824Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:27.118822378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:12:27.121978348Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:12:27.418820004Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:12:27.424221351Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:12:27.62772876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:12:27.630837247Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:12:28.097590275Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:12:28.097633778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:12:28.097644763Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:12:28.097643913Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:29.544072377Z caller=checkpoint.go:372 msg="writing series" size="245 kB"
level=debug ts=2026-04-28T22:12:30.152870995Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:12:31.518739806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:12:31.522699321Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:12:32.512524715Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:12:32.516441443Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:12:32.629674748Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:12:32.633275853Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:12:32.796875631Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T22:12:33.098049849Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:12:34.481677701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:12:34.490264382Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:12:36.049868272Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:12:37.429077828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:12:37.432687105Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=warn ts=2026-04-28T22:12:38.097160464Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:12:38.875398895Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:12:38.878824323Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:12:38.878843875Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:38.878866767Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:12:38.881756278Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:12:39.303141141Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-04-28T22:12:41.736747341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:12:41.742605622Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:12:42.555895205Z caller=checkpoint.go:372 msg="writing series" size="8.2 kB"
level=debug ts=2026-04-28T22:12:42.55591818Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=10
ts=2026-04-28T22:12:42.93070411Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:12:43.097594709Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:12:45.232734705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:12:45.236728362Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:12:45.509656503Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:12:45.513063711Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:12:45.513082694Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:45.513102799Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:12:45.516027409Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:12:45.80892358Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-28T22:12:46.296179453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:12:46.299553101Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:12:47.642212426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:12:47.646787855Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-28T22:12:48.097705937Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:49.062045788Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-04-28T22:12:50.32089323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:12:50.328864093Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:12:52.161147007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:12:52.167127575Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:12:52.315267184Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
level=warn ts=2026-04-28T22:12:53.097836604Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:12:55.571006279Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-28T22:12:55.571029937Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=4
level=info ts=2026-04-28T22:12:55.981217419Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:12:55.990218996Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:12:57.708193806Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:12:57.712046913Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:12:57.71206921Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:12:57.712090765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:12:57.71472804Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:12:57.758820847Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:12:57.761948644Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:12:57.846202475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:12:57.849432044Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:12:58.075074136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:12:58.078581152Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=error ts=2026-04-28T22:12:58.078765987Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:12:58.079745745Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-28T22:12:58.097401711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:12:58.097409004Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:12:58.09742865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:12:58.097436292Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:12:58.844665663Z caller=checkpoint.go:372 msg="writing series" size="19 MB"
level=debug ts=2026-04-28T22:12:58.84469188Z caller=checkpoint.go:384 msg="flushing series" totalSize="19 MB" series=1
level=info ts=2026-04-28T22:12:58.944989821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:12:58.948907772Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:13:00.03798114Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:13:02.073585867Z caller=checkpoint.go:372 msg="writing series" size="279 kB"
level=info ts=2026-04-28T22:13:02.214189953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:13:02.219909667Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:13:02.263967206Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:13:02.267698946Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:13:02.267720577Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:13:02.267747924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:13:02.271230565Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=warn ts=2026-04-28T22:13:03.097755272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:04.716690663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:13:04.721321685Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T22:13:04.721341141Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:13:04.72136392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:13:04.725290933Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:13:05.326178352Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=debug ts=2026-04-28T22:13:05.506530643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:13:07.112004035Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:13:07.115459981Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:13:08.097536738Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:08.224822244Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:13:08.232921389Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T22:13:08.580014484Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-04-28T22:13:09.445391136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:13:09.448777826Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:13:09.448800713Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:13:09.448822402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:13:09.451512227Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:13:09.628118415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:13:09.631741834Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:13:11.832444238Z caller=checkpoint.go:372 msg="writing series" size="2.9 kB"
level=info ts=2026-04-28T22:13:12.726439074Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:13:12.734772567Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T22:13:12.932673913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:13:13.097146105Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:13.290881008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:13:13.296175334Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:13:13.460244938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:13:13.463749161Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:13:13.657169215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:13:13.661080803Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:13:15.086181649Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T22:13:15.086210076Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T22:13:15.246733378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:13:15.250951672Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
ts=2026-04-28T22:13:15.851225252Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58548"
level=info ts=2026-04-28T22:13:16.511962583Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:13:16.515313627Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=info ts=2026-04-28T22:13:17.502204079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:13:17.506433518Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=warn ts=2026-04-28T22:13:18.097510272Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:13:18.338403212Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-04-28T22:13:19.984572775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:13:19.988077295Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:13:20.513460883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:13:20.516751809Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:13:20.615978353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:13:20.619167846Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:13:21.339178154Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:13:21.342484061Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:13:21.592499658Z caller=checkpoint.go:372 msg="writing series" size="247 kB"
level=info ts=2026-04-28T22:13:21.615358961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:13:21.618641256Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-28T22:13:23.098032606Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:24.475888274Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:13:24.479353232Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:13:24.845453028Z caller=checkpoint.go:372 msg="writing series" size="184 kB"
level=info ts=2026-04-28T22:13:25.538383139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:13:25.545277932Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:13:27.086553341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:13:27.091067021Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:13:28.097652354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:13:28.097679141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:13:28.097686984Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:13:28.097683911Z caller=checkpoint.go:384 msg="flushing series" totalSize="641 kB" series=3
level=warn ts=2026-04-28T22:13:28.097704311Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:28.105527591Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000017.tmp new=/var/loki/wal/checkpoint.000017
level=info ts=2026-04-28T22:13:28.107154554Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.009983012s
level=debug ts=2026-04-28T22:13:31.702747779Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T22:13:33.098118194Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:37.483732785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:13:37.490969801Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T22:13:38.097552503Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:42.215357892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:13:42.220693714Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
ts=2026-04-28T22:13:42.934308462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:13:43.097996904Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:43.617798898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:13:43.622094462Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:13:43.622117439Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:13:43.622145205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:13:43.625564247Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=info ts=2026-04-28T22:13:43.987365704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:13:43.995530673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:13:44.805848585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:13:44.813516146Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T22:13:45.853907251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:43736"
level=warn ts=2026-04-28T22:13:48.097743209Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:48.365283444Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:13:48.369972676Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:13:48.980159745Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:13:48.984211434Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:13:49.060249101Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:13:49.065372493Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:13:49.576384905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:13:49.579707707Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:13:51.108746898Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:13:51.113777185Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:13:52.692486251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:13:52.698029146Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T22:13:53.097249631Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:54.322900739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:13:54.327254679Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:13:55.079949704Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:13:55.084759971Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:13:58.079010771Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:13:58.079895994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:13:58.096746428Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T22:13:58.096876978Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000018
level=info ts=2026-04-28T22:13:58.096884535Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T22:13:58.096900501Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=11.688µs success=true
level=debug ts=2026-04-28T22:13:58.097126968Z caller=checkpoint.go:372 msg="writing series" size="5.7 kB"
level=warn ts=2026-04-28T22:13:58.097145337Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:13:58.097150724Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:13:58.097169782Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:13:58.097176936Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:13:58.279747133Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:13:59.913557511Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:14:00.387039383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:14:00.390849965Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:14:01.35044394Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=info ts=2026-04-28T22:14:01.783679123Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:14:01.787443156Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:14:02.089553119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:14:02.093350646Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:14:02.149493306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:14:02.153734841Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=warn ts=2026-04-28T22:14:03.097445362Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:14:04.716907944Z caller=checkpoint.go:372 msg="writing series" size="67 MB"
level=debug ts=2026-04-28T22:14:04.716976175Z caller=checkpoint.go:384 msg="flushing series" totalSize="67 MB" series=3
level=info ts=2026-04-28T22:14:07.416329097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:14:07.429955382Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=debug ts=2026-04-28T22:14:07.858468317Z caller=checkpoint.go:372 msg="writing series" size="744 kB"
level=info ts=2026-04-28T22:14:07.887560086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:14:07.898016343Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-28T22:14:08.097211392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:08.439991758Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:14:08.453295045Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:14:09.121057453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:14:09.128591783Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:14:11.117325319Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T22:14:11.117386684Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.5 MB" series=2
level=info ts=2026-04-28T22:14:11.238552712Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:14:11.254184492Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-28T22:14:12.936687734Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:14:13.097407926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:13.202157182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:14:13.215865738Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
ts=2026-04-28T22:14:14.356819725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:55618"
level=debug ts=2026-04-28T22:14:14.362653681Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
level=info ts=2026-04-28T22:14:14.461173492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:14:14.472120114Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T22:14:14.472152151Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:14:14.472181153Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:14:14.482280092Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:14:14.813755978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:14:14.822145367Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
ts=2026-04-28T22:14:15.856655393Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59510"
level=debug ts=2026-04-28T22:14:17.615805107Z caller=checkpoint.go:372 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T22:14:18.097150392Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:18.109238687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:14:18.1265215Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:14:20.86920826Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=warn ts=2026-04-28T22:14:23.098168789Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:14:24.126899152Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-28T22:14:24.12692437Z caller=checkpoint.go:384 msg="flushing series" totalSize="5.0 MB" series=4
level=info ts=2026-04-28T22:14:25.123588264Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:14:25.129631118Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:14:25.129663628Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:14:25.1296927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:14:25.133943658Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:14:27.312290579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:14:27.320651767Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:14:27.32067276Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:14:27.320698374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:14:27.32812772Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:14:27.374586942Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-04-28T22:14:28.097161598Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:14:28.097188623Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T22:14:28.097177524Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:28.097197091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:14:30.628282451Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=debug ts=2026-04-28T22:14:30.724780766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:14:30.805895645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:14:30.810358143Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:14:32.098314481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:14:32.101580118Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:14:33.097537805Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:33.791563545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:14:33.795552231Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:14:33.882156959Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T22:14:33.882180245Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=info ts=2026-04-28T22:14:34.861842089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:14:34.867037338Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:14:35.288663086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:14:35.292795588Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:14:35.878527934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:14:35.883735315Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:14:37.114958008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:14:37.122911114Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:14:37.133999125Z caller=checkpoint.go:372 msg="writing series" size="54 kB"
level=info ts=2026-04-28T22:14:37.53335678Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:14:37.540662778Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=warn ts=2026-04-28T22:14:38.09714821Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:14:38.191308679Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:14:38.348636949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="18 MB" avg_uncomp="18 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:14:38.357586693Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:14:39.604739413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:14:39.608219738Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:14:40.38696701Z caller=checkpoint.go:372 msg="writing series" size="208 kB"
ts=2026-04-28T22:14:40.43164884Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:47452"
ts=2026-04-28T22:14:42.940075359Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:14:43.097704203Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:43.414208217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:14:43.417678297Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-28T22:14:43.640031467Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
ts=2026-04-28T22:14:44.358998011Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:43678"
level=info ts=2026-04-28T22:14:44.877965019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:14:44.881284879Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:14:46.893038236Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T22:14:47.061374711Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:14:47.064870396Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:14:47.489474344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:14:47.49546896Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T22:14:48.09714871Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:14:49.770094054Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:14:49.774888917Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:14:50.148881717Z caller=checkpoint.go:372 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-28T22:14:50.148944405Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.1 MB" series=5
level=info ts=2026-04-28T22:14:50.176221108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:14:50.179660886Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:14:51.265901987Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:14:51.269548727Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:14:52.138912399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:14:52.145894272Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:14:52.145919779Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:14:52.145968644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:14:52.154344754Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T22:14:53.097670144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:14:53.398775619Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T22:14:56.237074084Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:14:56.242881946Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:14:56.671101545Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-04-28T22:14:56.671127123Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=debug ts=2026-04-28T22:14:57.054894924Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:14:57.38123776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:14:57.384749905Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:14:57.725670451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:14:57.729155159Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:14:58.010732765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=88334a9933abaaf2 immediate=false num_chunks=1 total_comp="641 B" avg_comp="641 B" total_uncomp="615 B" avg_uncomp="615 B" full=1 labels="{stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:14:58.014158265Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/88334a9933abaaf2/19dd5df5efb%3A19dd5df7a54%3Adfc48183?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"tenant-dev-rkcd\"}"
level=error ts=2026-04-28T22:14:58.014177373Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:14:58.014197513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="119 B" avg_comp="119 B" total_uncomp="204 kB" avg_uncomp="204 kB" idle=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:14:58.01753754Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-28T22:14:58.039596986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:14:58.042842917Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:14:58.078708054Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:14:58.079764585Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T22:14:58.097639071Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:14:58.097666563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:14:58.097676091Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:14:58.097698484Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:14:59.905561087Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=warn ts=2026-04-28T22:15:03.098017823Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:15:03.158197402Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=info ts=2026-04-28T22:15:04.406765141Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:15:04.410542122Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:15:05.731457476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:15:05.734866928Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:15:06.134144727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:15:06.142398251Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:15:06.411669484Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T22:15:06.411716896Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-04-28T22:15:08.097286035Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:08.468663971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:15:08.473977146Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-28T22:15:09.664236242Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-04-28T22:15:10.576842531Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:15:10.585340129Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:15:11.261163417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:15:11.264653293Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:15:12.91751037Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
ts=2026-04-28T22:15:12.942262091Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:15:13.097467758Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:14.141551773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:15:14.146955861Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T22:15:14.360656614Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57552"
level=info ts=2026-04-28T22:15:14.804869943Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:15:14.811290021Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:15:16.169873029Z caller=checkpoint.go:372 msg="writing series" size="295 kB"
level=warn ts=2026-04-28T22:15:18.098141518Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:15:19.423285878Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=debug ts=2026-04-28T22:15:22.676064455Z caller=checkpoint.go:372 msg="writing series" size="70 kB"
level=warn ts=2026-04-28T22:15:23.097393178Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:15:25.23379526Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:15:25.929847867Z caller=checkpoint.go:372 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T22:15:25.929870865Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.1 MB" series=6
level=debug ts=2026-04-28T22:15:26.426058752Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:15:28.097980394Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:15:28.098005896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:15:28.098015103Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:15:28.098023157Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:29.02160336Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="1.6 MB" avg_comp="1.6 MB" total_uncomp="13 MB" avg_uncomp="13 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:15:29.030007757Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd5d1032c%3A19dd60a34e6%3A6e4c16e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T22:15:29.030030505Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:15:29.030056416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:15:29.033740068Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-28T22:15:29.182427194Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T22:15:29.899484625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:15:29.903441246Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:15:29.903458728Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:15:29.903478537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:15:29.906837581Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:15:30.1396385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:15:30.143066977Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:15:31.190999326Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:15:31.196416459Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:15:31.281479488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:15:31.285073631Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:15:31.596887675Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:15:31.601020493Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:15:32.434727466Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T22:15:33.097781018Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:33.247137354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:15:33.25524847Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:15:35.5669309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:15:35.570461382Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:15:35.687897543Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T22:15:36.787093642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:15:36.791055404Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=info ts=2026-04-28T22:15:37.537344257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:15:37.541107002Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:15:37.664300776Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:15:37.667992787Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:15:38.097275734Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:15:38.940998116Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T22:15:39.60883243Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:15:39.615345593Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-28T22:15:40.699572984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:15:40.703470454Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:15:40.703490301Z caller=flush.go:240 component=ingester loop=20 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:15:40.70351251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:15:40.706400748Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:15:42.194094478Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
level=info ts=2026-04-28T22:15:42.249997416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:15:42.254539408Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-28T22:15:42.714764182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:15:42.719120001Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
ts=2026-04-28T22:15:42.944494172Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:15:43.097366711Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:43.876567835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:15:43.880787798Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:15:45.234592905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:15:45.242279457Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:15:45.446917954Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T22:15:46.938736774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:15:46.94250124Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:15:47.365853818Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:15:47.370899196Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-28T22:15:48.097383601Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:48.390829176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:15:48.394278741Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:15:48.700392592Z caller=checkpoint.go:372 msg="writing series" size="614 kB"
level=debug ts=2026-04-28T22:15:48.700418104Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=7
level=info ts=2026-04-28T22:15:51.64616475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:15:51.655120987Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:15:51.673704161Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:15:51.953328532Z caller=checkpoint.go:372 msg="writing series" size="68 kB"
level=warn ts=2026-04-28T22:15:53.097315357Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:53.447513015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:15:53.451220141Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:15:53.83873953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:15:53.842126266Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:15:55.205997112Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T22:15:57.935574609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:15:57.940832508Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:15:58.078960692Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:15:58.079406133Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=debug ts=2026-04-28T22:15:58.080780826Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T22:15:58.080836816Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974904 n=1
level=debug ts=2026-04-28T22:15:58.080896726Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974904 n=1
level=info ts=2026-04-28T22:15:58.097818544Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:15:58.097837357Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:15:58.097845455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:15:58.097861314Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:15:58.307610072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:15:58.311278554Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:15:58.459363378Z caller=checkpoint.go:372 msg="writing series" size="161 kB"
level=info ts=2026-04-28T22:16:00.711358737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:16:00.714886501Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:16:01.711980731Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T22:16:03.097945616Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:04.9650447Z caller=checkpoint.go:372 msg="writing series" size="567 kB"
level=info ts=2026-04-28T22:16:07.938739993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:16:07.945923883Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:16:07.945942747Z caller=flush.go:240 component=ingester loop=25 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:16:07.945968929Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:16:07.952015219Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:16:07.989443404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:16:07.993326537Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-28T22:16:08.097678414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:08.218373028Z caller=checkpoint.go:372 msg="writing series" size="807 kB"
level=debug ts=2026-04-28T22:16:08.218395821Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.6 MB" series=6
ts=2026-04-28T22:16:10.438029257Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:33220"
level=info ts=2026-04-28T22:16:11.261921426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:16:11.281841713Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:16:11.281864475Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:16:11.281890631Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:16:11.285715211Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:16:11.471148572Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
ts=2026-04-28T22:16:12.946326511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:16:13.097282115Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:13.489838395Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:16:13.493756653Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:16:14.619763496Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:16:14.627578291Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:16:14.723961973Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-28T22:16:17.977756551Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-04-28T22:16:18.098086283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:18.520159071Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:16:18.524110856Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:16:19.433205019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:16:19.436804614Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:16:19.436826619Z caller=flush.go:240 component=ingester loop=4 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:16:19.43685072Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:16:19.439824752Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:16:19.503175289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:16:19.506515655Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:16:19.506533037Z caller=flush.go:240 component=ingester loop=1 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:16:19.737057111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:16:19.741112074Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:16:21.23002329Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=info ts=2026-04-28T22:16:22.019214904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:16:22.023399346Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=warn ts=2026-04-28T22:16:23.098012772Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:23.727261433Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:16:23.732561597Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:16:23.890114953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:16:23.893538144Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:16:23.983812273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:16:23.987074294Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:16:24.483116205Z caller=checkpoint.go:372 msg="writing series" size="469 kB"
level=debug ts=2026-04-28T22:16:24.483139076Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-28T22:16:25.908038053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:16:25.912485671Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:16:26.244516782Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:16:26.249745571Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:16:27.55636612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:16:27.556444821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:16:27.560373186Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-28T22:16:27.642641Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:16:27.644229021Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T22:16:27.736629249Z caller=checkpoint.go:372 msg="writing series" size="44 kB"
level=info ts=2026-04-28T22:16:27.967330469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:16:27.971958797Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:16:28.097341535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:16:28.097349572Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:28.097372632Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:16:28.097382181Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:16:28.146424119Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:16:28.150501033Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:16:30.963304592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:16:30.967224316Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:16:30.989837702Z caller=checkpoint.go:372 msg="writing series" size="617 kB"
level=info ts=2026-04-28T22:16:31.546021087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:16:31.549515097Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:16:33.097305645Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:34.242024571Z caller=checkpoint.go:372 msg="writing series" size="406 kB"
level=debug ts=2026-04-28T22:16:34.242048934Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=3
level=info ts=2026-04-28T22:16:36.277625986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:16:36.286035436Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:16:36.447299601Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:16:36.452095879Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:16:37.485146771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:16:37.489956905Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:16:37.495027216Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=info ts=2026-04-28T22:16:37.721643933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:16:37.725062172Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=warn ts=2026-04-28T22:16:38.097630654Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:38.509215402Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:16:38.512344641Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-28T22:16:40.748225449Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=info ts=2026-04-28T22:16:41.046864938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:16:41.050464141Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
ts=2026-04-28T22:16:42.948091556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:16:43.097378271Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:44.001270855Z caller=checkpoint.go:372 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T22:16:44.702524118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:16:44.707003517Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:16:45.482470731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:16:45.490063437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T22:16:45.865300702Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:51580"
level=info ts=2026-04-28T22:16:46.231974885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:16:46.236288003Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-28T22:16:46.264346638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:16:46.267932908Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:16:47.254967309Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=warn ts=2026-04-28T22:16:48.097551048Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:48.274887195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:16:48.282028415Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:16:49.657665977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:16:49.661370713Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=debug ts=2026-04-28T22:16:50.507183684Z caller=checkpoint.go:372 msg="writing series" size="126 kB"
level=debug ts=2026-04-28T22:16:52.36538216Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T22:16:53.097461661Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:16:53.492658579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:16:53.498165092Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:16:53.760200309Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=debug ts=2026-04-28T22:16:57.013920234Z caller=checkpoint.go:372 msg="writing series" size="188 kB"
level=debug ts=2026-04-28T22:16:57.013942151Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-28T22:16:58.079145402Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:16:58.079150125Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:16:58.097553759Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T22:16:58.097549926Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:16:58.097576165Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:16:58.09758401Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:16:58.165905549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:16:58.171161984Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:16:59.376161049Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:16:59.380805532Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-28T22:16:59.401952742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:16:59.404945279Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:16:59.404961668Z caller=flush.go:240 component=ingester loop=17 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:16:59.404981209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:16:59.407752316Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:17:00.266422034Z caller=checkpoint.go:372 msg="writing series" size="194 kB"
level=info ts=2026-04-28T22:17:01.896129436Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:17:01.899737391Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=warn ts=2026-04-28T22:17:03.098063143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:03.421550354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:17:03.425203069Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:17:03.519510592Z caller=checkpoint.go:372 msg="writing series" size="80 kB"
level=info ts=2026-04-28T22:17:05.847838401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:17:05.853081962Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:17:06.77273548Z caller=checkpoint.go:372 msg="writing series" size="112 kB"
level=info ts=2026-04-28T22:17:07.918141302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:17:07.922955548Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-28T22:17:07.933987282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:17:07.937405582Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=warn ts=2026-04-28T22:17:08.098229132Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:08.558448793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:17:08.566279549Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:17:08.566301613Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:17:08.566332176Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:17:08.573475674Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:17:09.789330771Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:17:09.793368808Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:17:10.043724427Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T22:17:10.043749159Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=4
level=info ts=2026-04-28T22:17:10.102669207Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:17:10.106319697Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:17:10.793045881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:17:10.796582891Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:17:11.243556871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:17:12.057868957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:17:12.061152785Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:17:12.349507681Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:17:12.353398698Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-28T22:17:12.949663914Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:17:13.097509647Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:17:13.27832398Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
ts=2026-04-28T22:17:14.36888422Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:33834"
level=info ts=2026-04-28T22:17:15.615185905Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:17:15.620090937Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:17:16.531826295Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=warn ts=2026-04-28T22:17:18.09723476Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:19.089135688Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:17:19.093872198Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:17:19.784847249Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-28T22:17:23.002703868Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:17:23.037749366Z caller=checkpoint.go:372 msg="writing series" size="23 kB"
level=warn ts=2026-04-28T22:17:23.097939283Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:26.039469921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:17:26.047815312Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:17:26.290736683Z caller=checkpoint.go:372 msg="writing series" size="358 kB"
level=info ts=2026-04-28T22:17:27.692451306Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:17:27.700333977Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:17:28.097774862Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:17:28.097805319Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:17:28.097815016Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:17:28.097900833Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:29.139351836Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:17:29.144380003Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:17:29.253670741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:17:29.262275444Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:17:29.54322977Z caller=checkpoint.go:372 msg="writing series" size="269 kB"
level=info ts=2026-04-28T22:17:31.20785392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:17:31.211240645Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:17:32.44320382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:17:32.450496851Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:17:32.608016998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:17:32.611217954Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:17:32.784804632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:17:32.788374486Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:17:32.796440958Z caller=checkpoint.go:372 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T22:17:33.097272934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:35.011824155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:17:35.016154658Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:17:36.049313405Z caller=checkpoint.go:372 msg="writing series" size="3.9 kB"
level=info ts=2026-04-28T22:17:36.373738213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:17:36.37734231Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-28T22:17:38.097258759Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:17:39.302535471Z caller=checkpoint.go:372 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T22:17:39.302559473Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-28T22:17:41.192159006Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:17:41.195855628Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:17:42.555605922Z caller=checkpoint.go:372 msg="writing series" size="8.9 kB"
ts=2026-04-28T22:17:42.951762292Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=info ts=2026-04-28T22:17:43.019023527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:17:43.02280324Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:17:43.073029669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:17:43.077264741Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-28T22:17:43.09816442Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T22:17:44.370577465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:34070"
level=debug ts=2026-04-28T22:17:45.808949619Z caller=checkpoint.go:372 msg="writing series" size="13 kB"
level=info ts=2026-04-28T22:17:47.773771585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:17:47.779901222Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:17:47.969357753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:17:47.972875491Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T22:17:48.097402275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:17:48.773333021Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:17:48.913675056Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:17:48.917781186Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:17:49.061282457Z caller=checkpoint.go:372 msg="writing series" size="258 kB"
level=info ts=2026-04-28T22:17:49.141738891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:17:49.147051922Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:17:52.314879491Z caller=checkpoint.go:372 msg="writing series" size="59 kB"
level=warn ts=2026-04-28T22:17:53.097849414Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:17:55.569990004Z caller=checkpoint.go:372 msg="writing series" size="2.4 MB"
level=debug ts=2026-04-28T22:17:55.570020172Z caller=checkpoint.go:384 msg="flushing series" totalSize="2.7 MB" series=5
level=info ts=2026-04-28T22:17:57.614192245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:17:57.619120709Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:17:58.052397122Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:17:58.055780918Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:17:58.078997217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:17:58.079829116Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=warn ts=2026-04-28T22:17:58.097386956Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:17:58.097419409Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:17:58.097447898Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:17:58.097466276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:17:58.405508916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:17:58.410391292Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:17:58.849942107Z caller=checkpoint.go:372 msg="writing series" size="20 MB"
level=debug ts=2026-04-28T22:17:58.8499657Z caller=checkpoint.go:384 msg="flushing series" totalSize="20 MB" series=1
level=info ts=2026-04-28T22:17:58.955536159Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:17:58.963757978Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:18:02.074202131Z caller=checkpoint.go:372 msg="writing series" size="307 kB"
level=info ts=2026-04-28T22:18:02.676457259Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:18:02.679961577Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:18:03.097449359Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:03.800387452Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:18:03.803432594Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:18:04.29593971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:18:04.300099717Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-28T22:18:04.596814856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:18:04.601427495Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:18:05.30269268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:18:05.32636309Z caller=checkpoint.go:372 msg="writing series" size="654 B"
level=info ts=2026-04-28T22:18:07.374087289Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:18:07.382086354Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-28T22:18:08.097870713Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:08.10992058Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:18:08.114252871Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:18:08.579272649Z caller=checkpoint.go:372 msg="writing series" size="241 kB"
level=info ts=2026-04-28T22:18:09.152003195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:18:09.15542006Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:18:11.832698794Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-04-28T22:18:12.954087157Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:18:13.097933422Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:18:13.398712325Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T22:18:14.372196674Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:57960"
level=debug ts=2026-04-28T22:18:15.085965565Z caller=checkpoint.go:372 msg="writing series" size="836 kB"
level=debug ts=2026-04-28T22:18:15.08598933Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T22:18:16.769517448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:18:16.773778305Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:18:16.885109351Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:18:16.888981615Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:18:16.889001576Z caller=flush.go:240 component=ingester loop=10 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:18:16.889021884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:18:16.892493773Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=warn ts=2026-04-28T22:18:18.097136297Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:18:18.339163923Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
level=info ts=2026-04-28T22:18:19.349013089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:18:19.356133841Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-28T22:18:21.592336588Z caller=checkpoint.go:372 msg="writing series" size="250 kB"
level=warn ts=2026-04-28T22:18:23.097922934Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:18:24.84494026Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:18:26.233916702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:18:26.239451739Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:18:26.395220911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:18:26.402210588Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=info ts=2026-04-28T22:18:27.130349996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:18:27.139353576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:18:28.097953193Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:18:28.097951557Z caller=checkpoint.go:384 msg="flushing series" totalSize="646 kB" series=3
level=debug ts=2026-04-28T22:18:28.097977318Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:18:28.097985508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:18:28.097986275Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:28.105795868Z caller=checkpoint.go:599 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000018.tmp new=/var/loki/wal/checkpoint.000018
level=info ts=2026-04-28T22:18:28.107412252Z caller=checkpoint.go:670 msg="checkpoint done" time=4m30.010377654s
level=info ts=2026-04-28T22:18:28.126141439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:18:28.130508427Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:18:30.29840065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:18:30.301881162Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:18:31.486944263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:18:31.490795642Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:18:32.455301268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:18:32.459705911Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-28T22:18:33.098057908Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:33.336788007Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:18:33.340218256Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=info ts=2026-04-28T22:18:34.314667151Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:18:34.319722162Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:18:35.951525609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:18:35.95510315Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:18:37.717256386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:18:37.72073032Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=warn ts=2026-04-28T22:18:38.098011289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:38.689726934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:18:38.693486244Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
ts=2026-04-28T22:18:40.447300146Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:53910"
level=info ts=2026-04-28T22:18:41.926829192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:18:41.934370074Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T22:18:42.955976226Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:18:43.097365621Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:43.36548296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:18:43.369231403Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:18:43.369255661Z caller=flush.go:240 component=ingester loop=28 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:18:43.369281567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:18:43.372430381Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=debug ts=2026-04-28T22:18:45.017804498Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:18:45.612827323Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:18:45.61689159Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T22:18:46.669137329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:18:47.657020415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:18:47.66083567Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=warn ts=2026-04-28T22:18:48.097124462Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:49.292030855Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:18:49.295591861Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=info ts=2026-04-28T22:18:51.805365614Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:18:51.812592383Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:18:51.812636013Z caller=flush.go:240 component=ingester loop=19 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:18:51.812662111Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:18:51.817262786Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:18:52.368860078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:18:52.372951289Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:18:52.875444508Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:18:52.879115051Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T22:18:53.098113142Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:54.960328913Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:18:54.964100163Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:18:55.82018915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:18:55.823911738Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:18:55.823932041Z caller=flush.go:240 component=ingester loop=7 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:18:55.823953542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:18:55.826861252Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:18:57.175849254Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:18:57.179647571Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:18:58.078982953Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:18:58.079249745Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:18:58.097195627Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:18:58.097223336Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T22:18:58.097226728Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:18:58.097234821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T22:18:58.097245572Z caller=checkpoint.go:712 msg="starting checkpoint"
level=info ts=2026-04-28T22:18:58.097671832Z caller=checkpoint.go:336 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000019
level=info ts=2026-04-28T22:18:58.097684673Z caller=checkpoint.go:549 msg="tmp checkpoint cleanup starting"
level=info ts=2026-04-28T22:18:58.097725504Z caller=checkpoint.go:554 msg="tmp checkpoint cleanup done" duration=32.989µs success=true
level=debug ts=2026-04-28T22:18:58.098012462Z caller=checkpoint.go:372 msg="writing series" size="5.8 kB"
level=info ts=2026-04-28T22:18:59.38665275Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:18:59.390479416Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:19:00.674137251Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:19:00.681152599Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:19:00.990779213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:19:00.994917019Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=debug ts=2026-04-28T22:19:01.351441833Z caller=checkpoint.go:372 msg="writing series" size="15 kB"
level=warn ts=2026-04-28T22:19:03.097869613Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:03.619907801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:19:03.623995049Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-28T22:19:04.719400608Z caller=checkpoint.go:372 msg="writing series" size="68 MB"
level=debug ts=2026-04-28T22:19:04.719470659Z caller=checkpoint.go:384 msg="flushing series" totalSize="68 MB" series=3
level=debug ts=2026-04-28T22:19:07.859284299Z caller=checkpoint.go:372 msg="writing series" size="856 kB"
level=warn ts=2026-04-28T22:19:08.097606087Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:10.04598413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:19:10.068397875Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:19:10.752849632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:19:10.763973475Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:19:10.764008408Z caller=flush.go:240 component=ingester loop=11 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:19:10.764049492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:19:10.773530233Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:19:11.116423068Z caller=checkpoint.go:372 msg="writing series" size="3.7 MB"
level=debug ts=2026-04-28T22:19:11.118239969Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.6 MB" series=2
level=info ts=2026-04-28T22:19:11.326597736Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:19:11.34524129Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
ts=2026-04-28T22:19:12.95820453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:19:13.097405742Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:19:14.363671719Z caller=checkpoint.go:372 msg="writing series" size="193 kB"
ts=2026-04-28T22:19:14.37690468Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:35164"
level=info ts=2026-04-28T22:19:14.65495237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:19:14.667823941Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:19:15.25075277Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:19:15.27280576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T22:19:15.877518504Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40482"
level=info ts=2026-04-28T22:19:15.970128544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:19:15.983922112Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-28T22:19:17.210862338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:19:17.229529997Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:19:17.617317298Z caller=checkpoint.go:372 msg="writing series" size="152 kB"
level=info ts=2026-04-28T22:19:18.078265294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:19:18.086280353Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=warn ts=2026-04-28T22:19:18.097604418Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:20.615027008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:19:20.623768619Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:19:20.63539289Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:19:20.870008549Z caller=checkpoint.go:372 msg="writing series" size="24 kB"
level=info ts=2026-04-28T22:19:22.120409211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:19:22.125550827Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:19:22.125578876Z caller=flush.go:240 component=ingester loop=22 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:19:22.12560231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:19:22.128407444Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=warn ts=2026-04-28T22:19:23.097853377Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:23.937219517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:19:23.940640268Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=debug ts=2026-04-28T22:19:24.128008272Z caller=checkpoint.go:372 msg="writing series" size="4.6 MB"
level=debug ts=2026-04-28T22:19:24.128031764Z caller=checkpoint.go:384 msg="flushing series" totalSize="4.9 MB" series=4
level=info ts=2026-04-28T22:19:25.728973286Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:19:25.73273765Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:19:26.457688112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:19:26.466240602Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:19:27.375318848Z caller=checkpoint.go:372 msg="writing series" size="253 kB"
level=info ts=2026-04-28T22:19:27.757283584Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:19:27.762034584Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:19:28.043986321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:19:28.0475217Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:19:28.097883535Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:19:28.097908668Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:19:28.097916653Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:19:28.097932423Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:19:30.628483293Z caller=checkpoint.go:372 msg="writing series" size="42 kB"
level=info ts=2026-04-28T22:19:31.747506933Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:19:31.751549022Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=warn ts=2026-04-28T22:19:33.097792022Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:33.397520873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:19:33.401419863Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:19:33.401448015Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:19:33.401475299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:19:33.40486662Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:19:33.864391594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:19:33.868364147Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=debug ts=2026-04-28T22:19:33.882900819Z caller=checkpoint.go:372 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-28T22:19:33.882916165Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=3
level=info ts=2026-04-28T22:19:34.237744588Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:19:34.241755581Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=debug ts=2026-04-28T22:19:35.692984268Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:19:35.999904621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:19:36.005041064Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=info ts=2026-04-28T22:19:36.91602604Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:19:36.920332451Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:19:37.131130526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=debug ts=2026-04-28T22:19:37.134179075Z caller=checkpoint.go:372 msg="writing series" size="55 kB"
level=error ts=2026-04-28T22:19:37.13476821Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=warn ts=2026-04-28T22:19:38.097804139Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:19:40.387519879Z caller=checkpoint.go:372 msg="writing series" size="168 kB"
level=info ts=2026-04-28T22:19:40.965672346Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:19:40.969756597Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:19:42.127903759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:19:42.13165128Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
ts=2026-04-28T22:19:42.961934229Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:19:43.097819868Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:43.166991904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:19:43.170740824Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=debug ts=2026-04-28T22:19:43.64063485Z caller=checkpoint.go:372 msg="writing series" size="61 kB"
level=info ts=2026-04-28T22:19:45.187169602Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:19:45.192563201Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-28T22:19:45.315992486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:19:45.321179933Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:19:45.321224884Z caller=flush.go:240 component=ingester loop=16 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:19:45.321274204Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:19:45.325393463Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:19:46.55294954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:19:46.556633319Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:19:46.556656377Z caller=flush.go:240 component=ingester loop=23 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:19:46.556676706Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:19:46.559466695Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:19:46.893630934Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=debug ts=2026-04-28T22:19:47.184677018Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T22:19:48.09800407Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:19:50.148503307Z caller=checkpoint.go:372 msg="writing series" size="2.9 MB"
level=debug ts=2026-04-28T22:19:50.148528516Z caller=checkpoint.go:384 msg="flushing series" totalSize="3.2 MB" series=5
level=info ts=2026-04-28T22:19:52.926987237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:19:52.932255912Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=warn ts=2026-04-28T22:19:53.09801108Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:19:53.399320518Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=info ts=2026-04-28T22:19:54.198063486Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:19:54.20174508Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=info ts=2026-04-28T22:19:54.441395225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:19:54.449188934Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:19:56.672143598Z caller=checkpoint.go:372 msg="writing series" size="16 MB"
level=debug ts=2026-04-28T22:19:56.672165937Z caller=checkpoint.go:384 msg="flushing series" totalSize="16 MB" series=2
level=info ts=2026-04-28T22:19:57.68096544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:19:57.68503449Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:19:58.079039302Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:19:58.079485014Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:19:58.09729747Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:19:58.097317481Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:19:58.097326066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:19:58.097396289Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:19:58.356302985Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:19:58.360374794Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:19:59.907148743Z caller=checkpoint.go:372 msg="writing series" size="868 kB"
level=info ts=2026-04-28T22:20:01.392839734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:20:01.397651442Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=info ts=2026-04-28T22:20:02.679945278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:20:02.683719758Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=warn ts=2026-04-28T22:20:03.097794143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:03.159027249Z caller=checkpoint.go:372 msg="writing series" size="6.3 kB"
level=info ts=2026-04-28T22:20:03.463272065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:20:03.467029079Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:20:06.412577288Z caller=checkpoint.go:372 msg="writing series" size="779 kB"
level=debug ts=2026-04-28T22:20:06.412603194Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.7 MB" series=3
level=warn ts=2026-04-28T22:20:08.097760554Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:20:09.284825729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:20:09.292977198Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T22:20:09.665133853Z caller=checkpoint.go:372 msg="writing series" size="246 kB"
level=info ts=2026-04-28T22:20:10.373179856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:20:10.377301205Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
ts=2026-04-28T22:20:10.452321939Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:34266"
level=debug ts=2026-04-28T22:20:12.917994582Z caller=checkpoint.go:372 msg="writing series" size="209 kB"
ts=2026-04-28T22:20:12.964594275Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=info ts=2026-04-28T22:20:12.988788443Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:20:12.992845783Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=warn ts=2026-04-28T22:20:13.098094013Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:20:13.259442175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:20:13.26677722Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:20:13.605255136Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T22:20:14.381572979Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:41132"
level=debug ts=2026-04-28T22:20:16.170892151Z caller=checkpoint.go:372 msg="writing series" size="290 kB"
level=warn ts=2026-04-28T22:20:18.09807324Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:20:18.92988392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:20:18.934884331Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:20:19.242832556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:20:19.24621623Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=debug ts=2026-04-28T22:20:19.423873983Z caller=checkpoint.go:372 msg="writing series" size="8.7 kB"
level=info ts=2026-04-28T22:20:19.728769387Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:20:19.732226758Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:20:19.816441502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:20:19.820341358Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:20:20.159491501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:20:20.163887478Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:20:20.856067329Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:20:20.860437283Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=error ts=2026-04-28T22:20:20.860457119Z caller=flush.go:240 component=ingester loop=30 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:20:20.860480281Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:20:20.864087654Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:20:22.137157205Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:20:22.141193449Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:20:22.676390105Z caller=checkpoint.go:372 msg="writing series" size="90 kB"
level=warn ts=2026-04-28T22:20:23.097696352Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:20:24.052989924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:20:24.062887122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:20:24.24153567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="9.9 kB" avg_comp="9.9 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:20:24.24512144Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:20:25.279193114Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:20:25.283072512Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=debug ts=2026-04-28T22:20:25.930796065Z caller=checkpoint.go:372 msg="writing series" size="1.1 MB"
level=debug ts=2026-04-28T22:20:25.930818839Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.9 MB" series=6
level=info ts=2026-04-28T22:20:26.047136528Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=92ff7401f2c9ee9d immediate=false num_chunks=1 total_comp="201 kB" avg_comp="201 kB" total_uncomp="889 kB" avg_uncomp="889 kB" full=1 labels="{stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:20:26.051191632Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/92ff7401f2c9ee9d/19dd5e05ce2%3A19dd5ea765e%3Ad70b5e7f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"integration1-zxrw\"}"
level=error ts=2026-04-28T22:20:26.051213758Z caller=flush.go:240 component=ingester loop=29 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:20:26.051235385Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bdb4896fd4fb243d immediate=false num_chunks=1 total_comp="1.9 kB" avg_comp="1.9 kB" total_uncomp="188 kB" avg_uncomp="188 kB" idle=1 labels="{stream=\"build-e2e-cavu\"}"
level=error ts=2026-04-28T22:20:26.055329771Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bdb4896fd4fb243d/19dd5e01dec%3A19dd5f60cbe%3A5243f755?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cavu\"}"
level=info ts=2026-04-28T22:20:26.307982691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:20:26.311595719Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:20:28.098014372Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:20:28.09804404Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T22:20:28.098038636Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:20:28.098052161Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T22:20:28.600373058Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T22:20:28.602274139Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T22:20:29.031152359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:20:29.036975069Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=debug ts=2026-04-28T22:20:29.183261085Z caller=checkpoint.go:372 msg="writing series" size="10 kB"
level=info ts=2026-04-28T22:20:31.382856713Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:20:31.386944409Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:20:31.72188768Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:20:31.725665709Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=debug ts=2026-04-28T22:20:32.436298378Z caller=checkpoint.go:372 msg="writing series" size="5.5 kB"
level=warn ts=2026-04-28T22:20:33.0973754Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:35.688856631Z caller=checkpoint.go:372 msg="writing series" size="1.3 kB"
level=info ts=2026-04-28T22:20:36.181453716Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:20:36.184997947Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=info ts=2026-04-28T22:20:36.73082527Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:20:36.73512987Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:20:37.328942439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:20:37.332948092Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=warn ts=2026-04-28T22:20:38.0973812Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:38.94192965Z caller=checkpoint.go:372 msg="writing series" size="56 kB"
ts=2026-04-28T22:20:40.454671913Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55340"
level=info ts=2026-04-28T22:20:40.763960925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bdb4896fd4fb243d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-cavu\"}"
level=error ts=2026-04-28T22:20:40.767874034Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bdb4896fd4fb243d/19dd5e01dec%3A19dd5f60cbe%3A5243f755?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cavu\"}"
level=debug ts=2026-04-28T22:20:42.19600688Z caller=checkpoint.go:372 msg="writing series" size="597 kB"
ts=2026-04-28T22:20:42.966790962Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:20:43.097233117Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:45.448280734Z caller=checkpoint.go:372 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T22:20:46.881021911Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="593 B" avg_comp="593 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:20:46.884716959Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=warn ts=2026-04-28T22:20:48.097494348Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:48.701458097Z caller=checkpoint.go:372 msg="writing series" size="539 kB"
level=debug ts=2026-04-28T22:20:48.701480446Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=7
level=info ts=2026-04-28T22:20:49.201800954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:20:49.206783754Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:20:49.206801865Z caller=flush.go:240 component=ingester loop=6 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:20:49.206824015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:20:49.211283846Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:20:49.212415347Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:20:49.301726279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:20:49.304987744Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=info ts=2026-04-28T22:20:51.461940285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:20:51.470882807Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:20:51.954178571Z caller=checkpoint.go:372 msg="writing series" size="87 kB"
level=warn ts=2026-04-28T22:20:53.097310143Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:55.20729882Z caller=checkpoint.go:372 msg="writing series" size="892 B"
level=info ts=2026-04-28T22:20:55.807858018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:20:55.81211778Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=error ts=2026-04-28T22:20:55.812139792Z caller=flush.go:240 component=ingester loop=27 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:20:55.812166785Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:20:55.815123126Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=info ts=2026-04-28T22:20:57.549876881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:20:57.554111687Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-28T22:20:58.079042616Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:20:58.079050636Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:20:58.097663354Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:20:58.097693732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:20:58.097703394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:20:58.097720042Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:20:58.459976865Z caller=checkpoint.go:372 msg="writing series" size="162 kB"
level=info ts=2026-04-28T22:21:00.101968237Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:21:00.11014207Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
level=debug ts=2026-04-28T22:21:01.712973542Z caller=checkpoint.go:372 msg="writing series" size="25 kB"
level=info ts=2026-04-28T22:21:02.022742669Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:21:02.027459125Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:21:03.029177491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:21:03.034078546Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=warn ts=2026-04-28T22:21:03.097851249Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:04.966761998Z caller=checkpoint.go:372 msg="writing series" size="682 kB"
level=info ts=2026-04-28T22:21:05.328827997Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="561 B" avg_comp="561 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:21:05.33229149Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:21:05.504857373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:21:05.512730125Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:21:05.512748556Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:21:05.512777045Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:21:05.520215867Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:21:06.697876405Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:21:06.701550263Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:21:06.811787619Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:21:06.815141292Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:21:07.697959116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:21:07.701423672Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=warn ts=2026-04-28T22:21:08.097570005Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:08.219537895Z caller=checkpoint.go:372 msg="writing series" size="830 kB"
level=debug ts=2026-04-28T22:21:08.219561551Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.8 MB" series=6
level=info ts=2026-04-28T22:21:09.958670181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:21:09.964139302Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
ts=2026-04-28T22:21:10.457653214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.68:55240"
level=debug ts=2026-04-28T22:21:11.472320949Z caller=checkpoint.go:372 msg="writing series" size="2.2 kB"
level=info ts=2026-04-28T22:21:11.49428302Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:21:11.498987007Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:21:12.004023977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:21:12.008306552Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
ts=2026-04-28T22:21:12.968215059Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5dd8bc9747-qzg4q-021aaac5 10.128.0.86:7946"
level=warn ts=2026-04-28T22:21:13.097523463Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:13.792670253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:21:13.796444074Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
level=info ts=2026-04-28T22:21:13.93276253Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:21:13.937301853Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=info ts=2026-04-28T22:21:14.12096899Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:21:14.12565633Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
ts=2026-04-28T22:21:14.384999627Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:37036"
level=debug ts=2026-04-28T22:21:14.725544261Z caller=checkpoint.go:372 msg="writing series" size="1.8 kB"
level=info ts=2026-04-28T22:21:15.007410651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bdb4896fd4fb243d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-cavu\"}"
level=error ts=2026-04-28T22:21:15.011163041Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bdb4896fd4fb243d/19dd5e01dec%3A19dd5f60cbe%3A5243f755?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cavu\"}"
level=info ts=2026-04-28T22:21:15.08253178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:21:15.086303803Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=info ts=2026-04-28T22:21:17.458895148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=b69c168180c12b97 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.6 kB" avg_uncomp="3.6 kB" full=1 labels="{stream=\"mintmaker\"}"
level=error ts=2026-04-28T22:21:17.462054481Z caller=flush.go:262 component=ingester loop=23 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/b69c168180c12b97/19dd5d1735d%3A19dd5d1c60d%3Ab7e07ae?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"mintmaker\"}"
level=debug ts=2026-04-28T22:21:17.978236819Z caller=checkpoint.go:372 msg="writing series" size="578 kB"
level=warn ts=2026-04-28T22:21:18.097748351Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:19.230419152Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:21:19.238604427Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T22:21:19.86406737Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6f9c438629c503d0 immediate=false num_chunks=1 total_comp="348 kB" avg_comp="348 kB" total_uncomp="3.2 MB" avg_uncomp="3.2 MB" synced=1 labels="{stream=\"konflux-kyverno\"}"
level=error ts=2026-04-28T22:21:19.86780288Z caller=flush.go:262 component=ingester loop=16 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6f9c438629c503d0/19dd5d16660%3A19dd5f4cae5%3A1974a5d7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-kyverno\"}"
level=info ts=2026-04-28T22:21:20.18383518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0ac5aa54530c8322 immediate=false num_chunks=1 total_comp="464 kB" avg_comp="464 kB" total_uncomp="3.1 MB" avg_uncomp="3.1 MB" synced=1 labels="{stream=\"openshift-kueue-operator\"}"
level=error ts=2026-04-28T22:21:20.18797799Z caller=flush.go:262 component=ingester loop=2 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ac5aa54530c8322/19dd5d0fb38%3A19dd618b3d6%3A142eaf76?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kueue-operator\"}"
level=info ts=2026-04-28T22:21:20.413581576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=3056b50d91835711 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="36 kB" avg_uncomp="36 kB" full=1 labels="{stream=\"happy-path-managed\"}"
level=error ts=2026-04-28T22:21:20.416843957Z caller=flush.go:262 component=ingester loop=17 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/3056b50d91835711/19dd5e006f3%3A19dd5e169f5%3A2481d642?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"happy-path-managed\"}"
level=debug ts=2026-04-28T22:21:20.592275515Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T22:21:21.231001742Z caller=checkpoint.go:372 msg="writing series" size="202 kB"
level=warn ts=2026-04-28T22:21:23.098094763Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:23.394674862Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T22:21:24.484664669Z caller=checkpoint.go:372 msg="writing series" size="638 kB"
level=debug ts=2026-04-28T22:21:24.484686829Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.4 MB" series=5
level=info ts=2026-04-28T22:21:25.087178052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:21:25.0944441Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:21:27.084562838Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:21:27.091019312Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"
level=debug ts=2026-04-28T22:21:27.736815938Z caller=checkpoint.go:372 msg="writing series" size="72 kB"
level=info ts=2026-04-28T22:21:28.098027847Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:21:28.098055876Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:21:28.098066816Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:21:28.098070222Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:30.475273297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:21:30.478925143Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:21:30.478948635Z caller=flush.go:240 component=ingester loop=18 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:21:30.478970612Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:21:30.482029142Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-28T22:21:30.990765905Z caller=checkpoint.go:372 msg="writing series" size="551 kB"
level=info ts=2026-04-28T22:21:31.279438738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0de6b9d61f7ff3d4 immediate=false num_chunks=1 total_comp="1.6 kB" avg_comp="1.6 kB" total_uncomp="3.4 kB" avg_uncomp="3.4 kB" full=1 labels="{stream=\"openshift-dns\"}"
level=error ts=2026-04-28T22:21:31.283395553Z caller=flush.go:262 component=ingester loop=20 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/de6b9d61f7ff3d4/19dd5d1735d%3A19dd5ec3d06%3A90489f0e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-dns\"}"
level=warn ts=2026-04-28T22:21:33.097783501Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:34.24290929Z caller=checkpoint.go:372 msg="writing series" size="265 kB"
level=info ts=2026-04-28T22:21:35.144389391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:21:35.151635797Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:21:35.151657556Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:21:35.151710791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:21:35.158856906Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-28T22:21:37.343588991Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=132111dbe70a145b immediate=false num_chunks=1 total_comp="3.0 kB" avg_comp="3.0 kB" total_uncomp="11 kB" avg_uncomp="11 kB" full=1 labels="{stream=\"caching\"}"
level=error ts=2026-04-28T22:21:37.347250221Z caller=flush.go:262 component=ingester loop=27 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/132111dbe70a145b/19dd5d0fe0f%3A19dd5d17418%3A4d2de65a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"caching\"}"
level=debug ts=2026-04-28T22:21:37.495831208Z caller=checkpoint.go:372 msg="writing series" size="1.2 kB"
level=warn ts=2026-04-28T22:21:38.097120102Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:39.416206878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=164f302a95e3394b immediate=false num_chunks=1 total_comp="575 B" avg_comp="575 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"konflux-policies\"}"
level=error ts=2026-04-28T22:21:39.419807561Z caller=flush.go:262 component=ingester loop=11 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/164f302a95e3394b/19dd5d17e38%3A19dd5d17e39%3A643366c7?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"konflux-policies\"}"
level=debug ts=2026-04-28T22:21:40.748805771Z caller=checkpoint.go:372 msg="writing series" size="251 kB"
level=debug ts=2026-04-28T22:21:40.748830548Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.1 MB" series=5
level=info ts=2026-04-28T22:21:42.447142383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0d813253ea2e8dd9 immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.8 MB" total_uncomp="91 MB" avg_uncomp="18 MB" full=4 synced=1 labels="{stream=\"product-kubearchive\"}"
level=error ts=2026-04-28T22:21:42.454216328Z caller=flush.go:262 component=ingester loop=25 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/d813253ea2e8dd9/19dd5d1735d%3A19dd5fda4fc%3Ad3982c2f?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"product-kubearchive\"}"
ts=2026-04-28T22:21:42.970235128Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-6de2b4e5 10.130.0.68:7946"
level=warn ts=2026-04-28T22:21:43.097984072Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:44.001787522Z caller=checkpoint.go:372 msg="writing series" size="3.3 kB"
ts=2026-04-28T22:21:44.386488444Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.86:48324"
ts=2026-04-28T22:21:45.885841643Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:40452"
level=info ts=2026-04-28T22:21:45.906173009Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c0c7aca0f9d8c92 immediate=false num_chunks=1 total_comp="37 kB" avg_comp="37 kB" total_uncomp="204 kB" avg_uncomp="204 kB" full=1 labels="{stream=\"openshift-console-operator\"}"
level=error ts=2026-04-28T22:21:45.909678411Z caller=flush.go:262 component=ingester loop=18 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c0c7aca0f9d8c92/19dd5d10119%3A19dd5ed0ff6%3Adbd2a067?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-console-operator\"}"
level=debug ts=2026-04-28T22:21:47.255540292Z caller=checkpoint.go:372 msg="writing series" size="544 kB"
level=warn ts=2026-04-28T22:21:48.098106159Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:49.29550303Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=24 total_comp="67 MB" avg_comp="2.8 MB" total_uncomp="554 MB" avg_uncomp="23 MB" full=23 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-28T22:21:49.304127168Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19dd5d10270%3A19dd5e0658c%3A2e042f75?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 24, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-28T22:21:50.508797782Z caller=checkpoint.go:372 msg="writing series" size="190 kB"
level=debug ts=2026-04-28T22:21:51.358332282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T22:21:51.866608727Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=76ff92381e37fe8a immediate=false num_chunks=1 total_comp="10 kB" avg_comp="10 kB" total_uncomp="25 kB" avg_uncomp="25 kB" full=1 labels="{stream=\"chains-e2e-uasg\"}"
level=error ts=2026-04-28T22:21:51.870363756Z caller=flush.go:262 component=ingester loop=10 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/76ff92381e37fe8a/19dd5dfdbfa%3A19dd5e88e8d%3A4faeedd?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"chains-e2e-uasg\"}"
level=info ts=2026-04-28T22:21:52.807280607Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="535 kB" avg_comp="535 kB" total_uncomp="5.4 MB" avg_uncomp="5.4 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T22:21:52.812217036Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd5d106c3%3A19dd5f21fc4%3A9a0fa05a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T22:21:53.081550037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=51b60c1f7ddfaa53 immediate=false num_chunks=2 total_comp="3.6 MB" avg_comp="1.8 MB" total_uncomp="25 MB" avg_uncomp="12 MB" full=1 synced=1 labels="{stream=\"build-e2e-iome\"}"
level=error ts=2026-04-28T22:21:53.086670561Z caller=flush.go:262 component=ingester loop=19 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/51b60c1f7ddfaa53/19dd5e17b1d%3A19dd5f22ef9%3A6052e593?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"build-e2e-iome\"}"
level=warn ts=2026-04-28T22:21:53.097738106Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:21:53.761753164Z caller=checkpoint.go:372 msg="writing series" size="16 kB"
level=info ts=2026-04-28T22:21:54.111917068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=62152a12b4c21964 immediate=false num_chunks=1 total_comp="12 kB" avg_comp="12 kB" total_uncomp="60 kB" avg_uncomp="60 kB" full=1 labels="{stream=\"openshift-service-ca-operator\"}"
level=error ts=2026-04-28T22:21:54.115634326Z caller=flush.go:262 component=ingester loop=4 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/62152a12b4c21964/19dd5d10606%3A19dd5ed163d%3Aad532648?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-service-ca-operator\"}"
level=info ts=2026-04-28T22:21:55.576668701Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=2 total_comp="4.5 MB" avg_comp="2.2 MB" total_uncomp="31 MB" avg_uncomp="15 MB" synced=2 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T22:21:55.584718246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd5d0fc1a%3A19dd5f1cb3c%3A7431f940?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T22:21:56.414160464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:21:56.418251218Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:21:56.418268763Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:21:56.418292559Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:21:56.422296811Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=debug ts=2026-04-28T22:21:57.014259885Z caller=checkpoint.go:372 msg="writing series" size="195 kB"
level=debug ts=2026-04-28T22:21:57.145537683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T22:21:58.07915639Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T22:21:58.079186873Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.70:11211 count=1
level=info ts=2026-04-28T22:21:58.097893252Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T22:21:58.097916878Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T22:21:58.097924626Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T22:21:58.097923531Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:21:59.563853794Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=0eddc296593d10c0 immediate=false num_chunks=1 total_comp="2.0 kB" avg_comp="2.0 kB" total_uncomp="5.0 kB" avg_uncomp="5.0 kB" full=1 labels="{stream=\"openshift-network-diagnostics\"}"
level=error ts=2026-04-28T22:21:59.567021375Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/eddc296593d10c0/19dd5d1735d%3A19dd5d17e39%3Ae96ffd88?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-diagnostics\"}"
level=debug ts=2026-04-28T22:22:00.26750957Z caller=checkpoint.go:372 msg="writing series" size="203 kB"
level=debug ts=2026-04-28T22:22:00.267532419Z caller=checkpoint.go:384 msg="flushing series" totalSize="1.2 MB" series=6
level=info ts=2026-04-28T22:22:01.719947367Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=6d7e0aa892aede76 immediate=false num_chunks=1 total_comp="1.1 kB" avg_comp="1.1 kB" total_uncomp="2.8 kB" avg_uncomp="2.8 kB" full=1 labels="{stream=\"openshift-machine-config-operator\"}"
level=error ts=2026-04-28T22:22:01.723269732Z caller=flush.go:262 component=ingester loop=22 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/6d7e0aa892aede76/19dd5d1735d%3A19dd5d1c60d%3A7f880f15?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-machine-config-operator\"}"
level=info ts=2026-04-28T22:22:02.027891321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:22:02.031686769Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:22:02.03171583Z caller=flush.go:240 component=ingester loop=3 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T22:22:02.031742764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:22:02.03509715Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=warn ts=2026-04-28T22:22:03.098048144Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:22:03.520769855Z caller=checkpoint.go:372 msg="writing series" size="82 kB"
level=info ts=2026-04-28T22:22:04.218478961Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=bdb4896fd4fb243d immediate=false num_chunks=1 total_comp="54 kB" avg_comp="54 kB" total_uncomp="188 kB" avg_uncomp="188 kB" full=1 labels="{stream=\"build-e2e-cavu\"}"
level=error ts=2026-04-28T22:22:04.221944304Z caller=flush.go:262 component=ingester loop=29 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/bdb4896fd4fb243d/19dd5e01dec%3A19dd5f60cbe%3A5243f755?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-cavu\"}"
level=info ts=2026-04-28T22:22:05.222832848Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5c691e6ee98231fc immediate=false num_chunks=1 total_comp="543 B" avg_comp="543 B" total_uncomp="1.1 kB" avg_uncomp="1.1 kB" full=1 labels="{stream=\"openshift-network-console\"}"
level=error ts=2026-04-28T22:22:05.226478674Z caller=flush.go:262 component=ingester loop=28 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5c691e6ee98231fc/19dd5d1735d%3A19dd5d17e8b%3A329033eb?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-network-console\"}"
level=info ts=2026-04-28T22:22:06.227429245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=8a9380103298d8be immediate=false num_chunks=1 total_comp="498 kB" avg_comp="498 kB" total_uncomp="2.6 MB" avg_uncomp="2.6 MB" synced=1 labels="{stream=\"kube-system\"}"
level=error ts=2026-04-28T22:22:06.231609229Z caller=flush.go:262 component=ingester loop=30 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/8a9380103298d8be/19dd5d0fb4b%3A19dd602aa24%3A1bd99b18?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"kube-system\"}"
level=debug ts=2026-04-28T22:22:06.773807386Z caller=checkpoint.go:372 msg="writing series" size="118 kB"
level=warn ts=2026-04-28T22:22:08.097896364Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T22:22:09.313535863Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=46b403ef7f512ffa immediate=false num_chunks=1 total_comp="460 kB" avg_comp="460 kB" total_uncomp="2.8 MB" avg_uncomp="2.8 MB" synced=1 labels="{stream=\"openshift-image-registry\"}"
level=error ts=2026-04-28T22:22:09.317989773Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/46b403ef7f512ffa/19dd5d10295%3A19dd6243786%3A473fd443?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-image-registry\"}"
level=info ts=2026-04-28T22:22:09.873546469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=8 total_comp="19 MB" avg_comp="2.3 MB" total_uncomp="162 MB" avg_uncomp="20 MB" full=6 synced=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T22:22:09.883079512Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd5d10168%3A19dd5d178b2%3Ab6e1a32a?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T22:22:10.044948238Z caller=checkpoint.go:372 msg="writing series" size="14 MB"
level=debug ts=2026-04-28T22:22:10.044974652Z caller=checkpoint.go:384 msg="flushing series" totalSize="14 MB" series=3
level=info ts=2026-04-28T22:22:11.870853892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=71815992b469c167 immediate=false num_chunks=1 total_comp="6.1 kB" avg_comp="6.1 kB" total_uncomp="15 kB" avg_uncomp="15 kB" full=1 labels="{stream=\"build-e2e-zblh\"}"
level=error ts=2026-04-28T22:22:11.875211316Z caller=flush.go:262 component=ingester loop=7 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/71815992b469c167/19dd5e0f8ad%3A19dd5e31547%3A951546fa?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"build-e2e-zblh\"}"
ts=2026-04-28T22:22:12.971897647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-5586f77b45-vszfq-3639b9ef 10.130.0.67:7946"
level=warn ts=2026-04-28T22:22:13.098186167Z caller=ingester.go:503 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T22:22:13.279094621Z caller=checkpoint.go:372 msg="writing series" size="187 kB"
level=info ts=2026-04-28T22:22:13.488733065Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=464e1cb192ef7563 immediate=false num_chunks=1 total_comp="253 kB" avg_comp="253 kB" total_uncomp="1.2 MB" avg_uncomp="1.2 MB" full=1 labels="{stream=\"gitlab-rep-qljp\"}"
level=error ts=2026-04-28T22:22:13.492651065Z caller=flush.go:262 component=ingester loop=3 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/464e1cb192ef7563/19dd5e09c75%3A19dd5ec0544%3Aa3203806?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"gitlab-rep-qljp\"}"
level=info ts=2026-04-28T22:22:13.947306003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ce9e9bfccd600ac1 immediate=false num_chunks=1 total_comp="6.9 kB" avg_comp="6.9 kB" total_uncomp="34 kB" avg_uncomp="34 kB" full=1 labels="{stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=error ts=2026-04-28T22:22:13.951200349Z caller=flush.go:262 component=ingester loop=1 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ce9e9bfccd600ac1/19dd5d17e0b%3A19dd5ed2b4e%3A78708196?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-kube-storage-version-migrator-operator\"}"
level=info ts=2026-04-28T22:22:14.257187883Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=26fa559c079bb5e6 immediate=false num_chunks=1 total_comp="825 kB" avg_comp="825 kB" total_uncomp="4.3 MB" avg_uncomp="4.3 MB" synced=1 labels="{stream=\"openshift-multus\"}"
level=error ts=2026-04-28T22:22:14.262823885Z caller=flush.go:262 component=ingester loop=6 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/26fa559c079bb5e6/19dd5d0fc1a%3A19dd61d8636%3A95701d7b?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-multus\"}"