level=info ts=2026-04-20T12:12:27.719020035Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-20T12:12:27.719058431Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-20T12:12:27.719488815Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-20T12:12:27.72066828Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-20T12:12:27.721229521Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:12:27.721334624Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.129.0.65 inf=eth0
level=info ts=2026-04-20T12:12:27.72204843Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-d370e8cb
ts=2026-04-20T12:12:27.722128234Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-20T12:12:27.722141216Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.65 advertisePort=7946
level=debug ts=2026-04-20T12:12:27.722161042Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.129.0.65 advertisePort=7946
level=info ts=2026-04-20T12:12:27.72229175Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-20T12:12:27.727273013Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:12:27.727310853Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-20T12:12:27.728688572Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-20T12:12:27.728839555Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:12:27.729091042Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-20T12:12:27.72910682Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-20T12:12:27.729105061Z
level=debug ts=2026-04-20T12:12:27.729123558Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-20T12:12:27.729141484Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-20T12:12:27.729163539Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-20T12:12:27.729171307Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-20T12:12:27.729169928Z
level=debug ts=2026-04-20T12:12:27.729177984Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-04-20T12:12:27.730792274Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:12:27.730812227Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:12:27.7308259Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=8.536416ms
level=error ts=2026-04-20T12:12:27.730836093Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-20T12:12:27.730850533Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=debug ts=2026-04-20T12:12:27.731571214Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-20T12:12:27.731666013Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.129.0.65 inf=eth0
ts=2026-04-20T12:12:27.73325344Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:27.73328624Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-20T12:12:27.749234519Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-20T12:12:27.749294306Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-20T12:12:27.74935201Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-20T12:12:27.749368906Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-20T12:12:27.749372755Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=info ts=2026-04-20T12:12:27.749377495Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-20T12:12:27.749382426Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-20T12:12:27.749394206Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-20T12:12:27.749410961Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-20T12:12:27.749418959Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-20T12:12:27.749650672Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-20T12:12:27.749658285Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-20T12:12:27.749670091Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-20T12:12:27.749675678Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-20T12:12:27.749707992Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-20T12:12:27.749712736Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-20T12:12:27.749753025Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-20T12:12:27.749777901Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-20T12:12:27.749783728Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-20T12:12:27.749788623Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-20T12:12:27.749873169Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-20T12:12:27.749884154Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-20T12:12:27.749887569Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-20T12:12:27.749890389Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-20T12:12:27.749893886Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-20T12:12:27.7499182Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-20T12:12:27.749952409Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-20T12:12:27.750017642Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=102.122µs errors=false
level=info ts=2026-04-20T12:12:27.7500252Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-20T12:12:27.75010963Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=193.898µs errors=false
level=info ts=2026-04-20T12:12:27.750118062Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-20T12:12:27.750141717Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=224.691µs
level=info ts=2026-04-20T12:12:27.750198218Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-20T12:12:27.75021751Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-20T12:12:27.750240751Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-04-20T12:12:27.750683725Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-20T12:12:27.750695221Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=info ts=2026-04-20T12:12:27.75024148Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=8.699154702s
level=info ts=2026-04-20T12:12:27.7507352Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=debug ts=2026-04-20T12:12:27.750751767Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
level=info ts=2026-04-20T12:12:27.750792948Z caller=loki.go:599 msg="Loki started" startup_time=105.615211ms
ts=2026-04-20T12:12:29.64932668Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:29.649354327Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:12:32.751051772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:12:33.648970136Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:33.648994658Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:12:37.751653562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:12:39.112276815Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:12:39.112298765Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:12:42.751201014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:12:45.025202612Z caller=logging.go:137 msg="GET /ready (503) 81.046µs"
level=warn ts=2026-04-20T12:12:47.751725425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:12:52.751733874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:12:54.248324396Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.64:7946"
ts=2026-04-20T12:12:54.249711397Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.62:7946"
ts=2026-04-20T12:12:54.25153172Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.105:7946: dial tcp 10.128.0.105:7946: connect: connection refused"
ts=2026-04-20T12:12:54.252450586Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.103:7946"
ts=2026-04-20T12:12:54.254497939Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.102:7946: dial tcp 10.128.0.102:7946: connect: connection refused"
ts=2026-04-20T12:12:54.256035132Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.68:7946: dial tcp 10.130.0.68:7946: connect: connection refused"
level=info ts=2026-04-20T12:12:54.256051617Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.52519505s
level=debug ts=2026-04-20T12:12:55.024084636Z caller=logging.go:137 msg="GET /ready (503) 75.656µs"
level=info ts=2026-04-20T12:12:57.75165696Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-20T12:12:57.751687065Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=warn ts=2026-04-20T12:12:57.751680361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:12:57.751696025Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:13:02.75090157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:13:06.463895871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:13:06.463956429Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:13:06.471498282Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:13:07.751019566Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:13:07.967074278Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
ts=2026-04-20T12:13:09.057102199Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38702"
level=warn ts=2026-04-20T12:13:12.751371353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:13:17.751011378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:13:22.751032035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:13:24.113088817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:13:24.118630899Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:13:26.998885484Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:13:27.728655499Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:13:27.729731834Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:13:27.751080112Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:13:27.751099628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:13:27.751108605Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:13:27.751121021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:13:32.751741669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:13:37.751182857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:13:37.968138637Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:13:39.40749324Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:13:42.751749582Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:13:47.665079499Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:13:47.675078264Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:13:47.751303919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:13:51.175330019Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:13:52.751676152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:13:55.968282983Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53336"
level=info ts=2026-04-20T12:13:57.750732327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:13:57.750806407Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:13:57.750815207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:13:57.750812878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:02.750846676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:14:03.869133465Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:14:07.171054798Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:14:07.75080766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:14:07.969309845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=warn ts=2026-04-20T12:14:12.751191543Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:17.75163455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:22.751542355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:14:24.285981938Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:14:27.728464317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:14:27.729514012Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:14:27.750869369Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:14:27.750888639Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:14:27.750899143Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:14:27.750914638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:32.750826716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:14:33.251864759Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:14:36.554000409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:14:36.568051603Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:14:37.751223071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:14:37.970747227Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=warn ts=2026-04-20T12:14:42.751407805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:47.751134167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:14:52.750948939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:14:57.751637063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:14:57.751664247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:14:57.751673Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:14:57.751689986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:14:59.011752229Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:14:59.013367311Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:15:02.751058361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:15:07.751614125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:15:07.971529317Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:15:08.476735871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:15:08.47681621Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:15:08.482011616Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:15:12.750820283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:15:17.751219788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:15:20.042807517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:15:20.05020302Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:15:22.751586612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:15:24.188985285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:15:24.196047731Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T12:15:25.973911803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:35078"
level=debug ts=2026-04-20T12:15:27.728582039Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:15:27.729630809Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:15:27.730900556Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T12:15:27.730940837Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974096 n=1
level=debug ts=2026-04-20T12:15:27.730979765Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974096 n=1
level=info ts=2026-04-20T12:15:27.75099409Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:15:27.751012966Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:15:27.751023013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:15:27.75102865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:15:32.750902568Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:15:34.402728048Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:15:35.687711077Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:15:37.751347602Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:15:37.972844342Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=warn ts=2026-04-20T12:15:42.750879017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:15:47.750938094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:15:52.751240193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:15:56.069579919Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:15:56.07729278Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-20T12:15:57.751585103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:15:57.751613844Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:15:57.7516223Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:15:57.751617506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:16:02.416041652Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:16:02.75119821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:07.751398472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:16:07.974844686Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:16:09.645366934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:16:09.653850907Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:16:12.751524296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:17.751611532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:22.751171525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:16:25.976842402Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42054"
level=info ts=2026-04-20T12:16:27.728908284Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:16:27.729384677Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:16:27.751693933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:16:27.751715178Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:16:27.751723531Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:16:27.751726181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:16:30.106577091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:16:32.751362573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:37.751632214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:16:37.976473592Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:16:39.979502851Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-20T12:16:42.751711115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:47.750995928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:16:52.750886879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:16:55.330238971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:16:55.337406987Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-20T12:16:57.750990044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:16:57.751027776Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:16:57.751036095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:16:57.751029157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:17:01.077637118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:17:01.153437369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:33446"
level=warn ts=2026-04-20T12:17:02.751204678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:17:04.265744292Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:17:04.272459823Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:17:07.751626451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:17:07.977953017Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=warn ts=2026-04-20T12:17:12.751591719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:17:17.751148638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:17:22.751589457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:17:27.729013367Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:17:27.729496101Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:17:27.750846461Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:17:27.750868169Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:17:27.750877107Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:17:27.750883143Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-20T12:17:27.750914537Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:17:27.750998123Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-20T12:17:27.751226139Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-20T12:17:32.107098859Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=warn ts=2026-04-20T12:17:32.751246172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:17:32.88763992Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:17:36.461097233Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T12:17:37.751007726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:17:37.980376836Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:17:40.816634809Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-20T12:17:42.751072233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:17:45.051485956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:17:45.056226715Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:17:45.171522504Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:17:47.75158406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:17:48.08652347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:17:49.526328742Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:17:52.06555932Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:17:52.072903897Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:17:52.75117989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:17:53.881055273Z caller=checkpoint.go:378 msg="writing series" size="1.6 kB"
ts=2026-04-20T12:17:55.981127425Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60264"
level=info ts=2026-04-20T12:17:57.751319411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:17:57.751346426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:17:57.751354408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:17:57.751350962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:17:58.235864994Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-20T12:17:58.64601295Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:18:01.159549779Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:41244"
level=debug ts=2026-04-20T12:18:02.590845919Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:18:02.751145961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:06.945118763Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:18:07.751674282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:18:07.983501218Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:18:09.078034295Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41756"
level=debug ts=2026-04-20T12:18:11.300508578Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=warn ts=2026-04-20T12:18:12.751449792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:15.655174495Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-20T12:18:17.751483255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:20.009589076Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=warn ts=2026-04-20T12:18:22.751781931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:24.364520667Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-04-20T12:18:24.364541971Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=14
level=info ts=2026-04-20T12:18:27.728899288Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:18:27.728898038Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:18:27.751307853Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:18:27.751326293Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:18:27.751334167Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:18:27.751339115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:28.719890801Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=debug ts=2026-04-20T12:18:29.62018787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:18:31.270672928Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:18:31.275588359Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:18:32.751011156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:33.074197646Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-04-20T12:18:36.704566409Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:18:36.710798366Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:18:37.433857612Z caller=checkpoint.go:378 msg="writing series" size="5.1 MB"
level=debug ts=2026-04-20T12:18:37.433880808Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.6 MB" series=3
level=warn ts=2026-04-20T12:18:37.751621554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:18:37.986656509Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:18:41.78389717Z caller=checkpoint.go:378 msg="writing series" size="8.3 kB"
level=warn ts=2026-04-20T12:18:42.751582148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:44.483160011Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-20T12:18:46.138427009Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:18:47.751612336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:50.493945574Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-20T12:18:52.75132648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:54.848965383Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-20T12:18:55.985243391Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:43086"
level=info ts=2026-04-20T12:18:57.750854712Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:18:57.750882914Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:18:57.750890966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:18:57.750885752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:18:57.977773007Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:18:59.202960356Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
ts=2026-04-20T12:19:01.16496455Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:56866"
level=warn ts=2026-04-20T12:19:02.751519221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:03.557909032Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T12:19:07.751106529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:07.913516926Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
ts=2026-04-20T12:19:07.989680344Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:19:12.268381036Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=warn ts=2026-04-20T12:19:12.751494953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:16.622676861Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=info ts=2026-04-20T12:19:17.729451908Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:19:17.73493592Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:19:17.751087371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:20.97738593Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-20T12:19:22.750812747Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:25.332794204Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=debug ts=2026-04-20T12:19:27.728420998Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:19:27.729476366Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:19:27.75080238Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:19:27.750805441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:27.750825954Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:19:27.750834303Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:19:29.687310406Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=info ts=2026-04-20T12:19:30.331689047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:19:30.339132014Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:19:30.33915147Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:19:30.33917339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:19:30.345059437Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:19:31.249999264Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:19:32.751003261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:34.041814002Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-20T12:19:37.751567826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:19:37.992085485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:19:38.396530292Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-20T12:19:38.991592987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-20T12:19:38.993478113Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:19:40.358443188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.6 MB" avg_comp="2.6 MB" total_uncomp="22 MB" avg_uncomp="22 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:19:40.365560422Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:19:42.751694558Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:42.751736188Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=debug ts=2026-04-20T12:19:47.106300013Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-20T12:19:47.751534444Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:51.461372015Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-20T12:19:52.750884969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:55.816356197Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
ts=2026-04-20T12:19:55.987883291Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:53462"
level=info ts=2026-04-20T12:19:57.751075453Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:19:57.751079117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:19:57.75110097Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:19:57.751109467Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:20:00.170903549Z caller=checkpoint.go:378 msg="writing series" size="307 kB"
level=debug ts=2026-04-20T12:20:00.170927903Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=19
ts=2026-04-20T12:20:01.170920041Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:60398"
level=warn ts=2026-04-20T12:20:02.751513412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:03.116116998Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:20:03.116168117Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:20:03.122962018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:20:04.525621997Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:20:07.751418235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:20:07.994171616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:20:08.880897191Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-04-20T12:20:11.770283744Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:20:11.777588675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:20:12.751020585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:13.235104442Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=info ts=2026-04-20T12:20:14.177407739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:20:14.181190701Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:20:16.441298219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:20:16.446241262Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:20:17.590578823Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:20:17.751345033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:21.94557827Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-20T12:20:22.751412081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:20:25.988999221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:39304"
level=debug ts=2026-04-20T12:20:26.300068002Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=info ts=2026-04-20T12:20:27.728988185Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:20:27.729462849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:20:27.750729643Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:20:27.750744394Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:20:27.750752543Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:20:27.750785619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:30.387212467Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:20:30.656170313Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-20T12:20:30.65619648Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=7
ts=2026-04-20T12:20:31.174283715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:38262"
level=debug ts=2026-04-20T12:20:31.914286592Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:20:32.751450704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:35.009395255Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-20T12:20:37.751630788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:20:37.995665929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:20:39.085647965Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:32838"
level=debug ts=2026-04-20T12:20:39.364230166Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=info ts=2026-04-20T12:20:39.998010525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:20:40.002696394Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:20:42.751122355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:43.719543089Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=warn ts=2026-04-20T12:20:47.75114193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:48.074243479Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-20T12:20:52.429659045Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-20T12:20:52.751683746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:56.784394759Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-20T12:20:57.751006958Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:20:57.751019178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:20:57.751040069Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:20:57.751050837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:21:00.995580738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:21:01.139271858Z caller=checkpoint.go:378 msg="writing series" size="299 kB"
level=warn ts=2026-04-20T12:21:02.751534301Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:05.49391889Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:21:07.750873793Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:21:07.998119407Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:21:08.562056106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:21:08.570899476Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T12:21:09.08708109Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:60030"
level=debug ts=2026-04-20T12:21:09.848983081Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-20T12:21:12.751588112Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:14.203791005Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-04-20T12:21:14.703234002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:21:14.70821813Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:21:17.75123586Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:18.558151392Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-04-20T12:21:20.0424098Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:21:22.751657255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:22.912984531Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:21:24.562876177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:21:24.566422572Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:21:25.273622607Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:21:25.992664789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48248"
level=debug ts=2026-04-20T12:21:27.267487144Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-04-20T12:21:27.728542742Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:21:27.72958264Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:21:27.750931833Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:21:27.750950347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:21:27.750958522Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:21:27.750979955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:31.623130705Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:21:32.751030015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:35.977581681Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=debug ts=2026-04-20T12:21:35.977606099Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-04-20T12:21:37.751353955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:21:38.000327931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:21:40.332323619Z caller=checkpoint.go:378 msg="writing series" size="304 kB"
level=warn ts=2026-04-20T12:21:42.750830913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:44.687313292Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-20T12:21:47.750841329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:49.041634447Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-20T12:21:52.75102997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:21:53.396634883Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-20T12:21:57.751013663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:21:57.751013381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:21:57.751042202Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:21:57.751050613Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:21:57.752076305Z caller=checkpoint.go:390 msg="flushing series" totalSize="314 kB" series=4
level=info ts=2026-04-20T12:21:57.757665944Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-20T12:21:57.761417224Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010291142s
level=debug ts=2026-04-20T12:21:59.108026031Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:22:00.854590765Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:22:00.862118572Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:22:02.750886274Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:22:04.646111095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:22:04.650649117Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:22:04.65066881Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:22:04.650691927Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:22:04.654657986Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:22:07.75081216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:22:08.001608561Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=info ts=2026-04-20T12:22:11.797095415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:22:11.801173018Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:22:12.751460734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:16.681347912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:22:17.750836895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:22:22.750956663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:22:23.673533857Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:22:23.681916789Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:22:24.069349387Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:22:27.728936507Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:22:27.7293289Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:22:27.750599978Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:22:27.750717456Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-20T12:22:27.750727333Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:22:27.750747159Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:22:27.75077066Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:22:27.750829222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:27.750949262Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-20T12:22:32.106018702Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-04-20T12:22:32.751584597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:36.461551178Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T12:22:37.750969754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:22:38.002722022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:22:40.815625584Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-20T12:22:42.751675095Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:45.171288332Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:22:47.750817299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:49.525338923Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:22:52.751369509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:22:53.560737392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:22:53.567837876Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:22:53.881846263Z caller=checkpoint.go:378 msg="writing series" size="1.9 kB"
level=info ts=2026-04-20T12:22:57.751635407Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:22:57.751648863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:22:57.751662468Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:22:57.751670658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:22:58.235438912Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-20T12:22:58.387422452Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:22:59.740557642Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:22:59.751322085Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:23:00.684560199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:23:00.688303129Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:23:02.589688016Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:23:02.750960479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:06.944527111Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:23:07.751032783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:23:08.003805406Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:23:09.180168207Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:23:11.299955185Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=warn ts=2026-04-20T12:23:12.750952982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:15.654938936Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-20T12:23:17.750901039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:20.009487615Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=debug ts=2026-04-20T12:23:20.00951106Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=debug ts=2026-04-20T12:23:22.655000399Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:23:22.751441772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:24.364212269Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-04-20T12:23:27.729219142Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:23:27.729816098Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:23:27.75113929Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:23:27.751155274Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:23:27.751162797Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:23:27.751180693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:28.719529504Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-20T12:23:32.751598885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:33.073957126Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=info ts=2026-04-20T12:23:35.31521742Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:23:35.324364495Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:23:37.435362259Z caller=checkpoint.go:378 msg="writing series" size="5.7 MB"
level=debug ts=2026-04-20T12:23:37.435388114Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.2 MB" series=4
level=warn ts=2026-04-20T12:23:37.751400561Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:23:38.005107707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:23:41.783565651Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:23:42.75081262Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:46.138965563Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-20T12:23:47.751204188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:50.099228514Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:23:50.493375049Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-20T12:23:52.751018302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:54.848269405Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-20T12:23:56.001733575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42424"
level=info ts=2026-04-20T12:23:57.003120687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:23:57.012068663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:23:57.147640414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:23:57.151278477Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:23:57.751139788Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:23:57.751166542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:23:57.751175002Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:23:57.751185684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:23:59.203334363Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-20T12:24:02.75159467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:03.557951256Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-20T12:24:07.751543191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:07.912728055Z caller=checkpoint.go:378 msg="writing series" size="70 kB"
ts=2026-04-20T12:24:08.006169815Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
ts=2026-04-20T12:24:09.100141888Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:45030"
level=debug ts=2026-04-20T12:24:10.891247126Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:24:10.892862301Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:24:12.267609121Z caller=checkpoint.go:378 msg="writing series" size="152 kB"
level=warn ts=2026-04-20T12:24:12.751539538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:16.085692876Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:24:16.622263173Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=info ts=2026-04-20T12:24:16.632364427Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:24:16.637021056Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:24:17.751478716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:20.97694832Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-20T12:24:22.750937965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:25.332471056Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-20T12:24:27.729300343Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:24:27.729863234Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:24:27.75117961Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:24:27.751199836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:24:27.751208837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:24:27.751204681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:29.687098242Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=warn ts=2026-04-20T12:24:32.75127694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:34.041303431Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-20T12:24:37.751274455Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:24:38.007204321Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:24:38.396329676Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T12:24:42.751136001Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:42.751166465Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=debug ts=2026-04-20T12:24:43.230519336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:24:47.106513329Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-20T12:24:47.106540197Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-04-20T12:24:47.751267863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:24:48.270979804Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:24:48.280802973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:24:48.375067219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:24:48.378871166Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:24:51.461666355Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-20T12:24:52.751223827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:55.815919848Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-20T12:24:57.751123767Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:24:57.751129914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:24:57.751151212Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:24:57.751158896Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:25:00.17072377Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=debug ts=2026-04-20T12:25:00.476697111Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:25:02.751001258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:04.525818204Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:25:07.750799783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:25:08.008629966Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:25:08.882861095Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
ts=2026-04-20T12:25:09.104150829Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:43096"
level=info ts=2026-04-20T12:25:11.913621935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:25:11.919113187Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:25:12.751368204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:13.235027969Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=debug ts=2026-04-20T12:25:13.813191597Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:25:17.589684358Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:25:17.751220889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:21.944981074Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-20T12:25:22.750805812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:26.300381287Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-20T12:25:27.729580411Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:25:27.730088364Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:25:27.751388801Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:25:27.751407547Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:25:27.751415376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:25:27.75142139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:30.656455361Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-20T12:25:30.656478574Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=10
level=warn ts=2026-04-20T12:25:32.751544346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:25:33.573877999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:25:33.578116765Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:25:35.009986891Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-20T12:25:37.751222553Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:25:38.011048647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:25:38.39429225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:25:38.40161521Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T12:25:39.105618421Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:47162"
level=debug ts=2026-04-20T12:25:39.364660897Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-20T12:25:42.750935848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:43.719826052Z caller=checkpoint.go:378 msg="writing series" size="363 kB"
level=debug ts=2026-04-20T12:25:46.761288996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:25:47.751028782Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:48.074441978Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-20T12:25:52.429216547Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-20T12:25:52.75167631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:53.076031898Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:25:54.722792686Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:25:54.727285677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-20T12:25:56.009824209Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60744"
level=debug ts=2026-04-20T12:25:56.783491555Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-20T12:25:57.751199864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:25:57.751199117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:25:57.751224838Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:25:57.751234655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:26:01.139221844Z caller=checkpoint.go:378 msg="writing series" size="356 kB"
level=warn ts=2026-04-20T12:26:02.750944635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:05.492904168Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:26:07.751619779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:26:08.013602013Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:26:09.848512092Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-20T12:26:12.751014638Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:14.203301487Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=info ts=2026-04-20T12:26:16.601027397Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:26:16.608110267Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:26:16.608129395Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:26:16.608150255Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:26:16.611651413Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:26:17.378800874Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:26:17.750796309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:18.557539621Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-04-20T12:26:18.557564898Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-20T12:26:19.343060636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:26:19.349665544Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:26:19.349684916Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:26:19.349706918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:26:19.355649949Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:26:22.75100194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:22.91225183Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
ts=2026-04-20T12:26:26.010958873Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:36030"
level=debug ts=2026-04-20T12:26:27.267384528Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=debug ts=2026-04-20T12:26:27.728513569Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:26:27.729654896Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:26:27.750985437Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:26:27.751008161Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:26:27.751015813Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:26:27.751040588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:30.141608034Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-20T12:26:31.209295216Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:46780"
level=debug ts=2026-04-20T12:26:31.622838289Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-20T12:26:32.751007993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:26:35.595691931Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:26:35.59940805Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:26:35.977482288Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=info ts=2026-04-20T12:26:36.979328094Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:26:36.986500071Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:26:37.751405727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:26:38.01471417Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:26:40.332111184Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=warn ts=2026-04-20T12:26:42.751104361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:44.686952501Z caller=checkpoint.go:378 msg="writing series" size="4.9 kB"
level=debug ts=2026-04-20T12:26:47.123939575Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:26:47.750807007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:49.041332253Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-04-20T12:26:52.201962957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:26:52.214564938Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:26:52.751160467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:53.396988849Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-20T12:26:56.012164196Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:52140"
level=debug ts=2026-04-20T12:26:57.751588956Z caller=checkpoint.go:390 msg="flushing series" totalSize="506 kB" series=8
level=info ts=2026-04-20T12:26:57.751631903Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:26:57.751630285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:26:57.751652723Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:26:57.751661874Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:26:57.756947559Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-20T12:26:57.758214359Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007360457s
level=info ts=2026-04-20T12:27:02.53167113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:27:02.538277139Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:27:02.751017508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:07.301703472Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:27:07.751476661Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:27:08.01599608Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=warn ts=2026-04-20T12:27:12.751456176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:27:13.613309759Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:27:13.620699015Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:27:17.75146176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:27:22.75153392Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:23.007174153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:27:26.013142639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:45492"
level=info ts=2026-04-20T12:27:27.729259081Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:27:27.729795416Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:27:27.751096944Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:27:27.751118201Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:27:27.751126522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:27.751138441Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:27:27.75114719Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:27:27.751237885Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-20T12:27:27.751465354Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-20T12:27:32.107315592Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-20T12:27:32.750968378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:27:35.868337248Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:27:35.8755464Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:27:36.462153031Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T12:27:37.750843935Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:27:38.01820155Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:27:40.816111416Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-20T12:27:42.751486998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:45.171298559Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:27:47.751281394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:49.525829808Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:27:52.750945806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:53.440906981Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:27:53.881135441Z caller=checkpoint.go:378 msg="writing series" size="2.1 kB"
level=info ts=2026-04-20T12:27:57.379956861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:27:57.385939441Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:27:57.751292828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:27:57.751295305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:27:57.751317531Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:27:57.751325346Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:27:57.794479882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:27:57.800982522Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:27:58.236166419Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
ts=2026-04-20T12:28:01.216731765Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:49968"
level=debug ts=2026-04-20T12:28:02.590815947Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:28:02.751339569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:03.551390591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:28:03.552960289Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:28:06.945782001Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:28:07.751804105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:28:08.020936573Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:28:11.300455895Z caller=checkpoint.go:378 msg="writing series" size="455 kB"
level=warn ts=2026-04-20T12:28:12.751694509Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:15.654986888Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-20T12:28:17.751348275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:20.012642926Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-20T12:28:20.012672896Z caller=checkpoint.go:390 msg="flushing series" totalSize="3.4 MB" series=13
level=warn ts=2026-04-20T12:28:22.751044663Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:23.755292152Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:28:24.36517078Z caller=checkpoint.go:378 msg="writing series" size="134 kB"
level=info ts=2026-04-20T12:28:27.729019394Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:28:27.730122966Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:28:27.751519979Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:28:27.751545841Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:28:27.751563508Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:28:27.751565436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:28.720228549Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=info ts=2026-04-20T12:28:30.675077327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:28:30.683173862Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:28:30.683192502Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:28:30.683214718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:28:30.688849757Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-20T12:28:31.219805887Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42786"
level=warn ts=2026-04-20T12:28:32.751598212Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:33.074970978Z caller=checkpoint.go:378 msg="writing series" size="285 kB"
level=debug ts=2026-04-20T12:28:37.43420395Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-04-20T12:28:37.434409993Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.9 MB" series=4
level=warn ts=2026-04-20T12:28:37.751539263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:28:38.023689117Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:28:39.114972462Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41266"
level=debug ts=2026-04-20T12:28:41.784194275Z caller=checkpoint.go:378 msg="writing series" size="24 kB"
level=warn ts=2026-04-20T12:28:42.751704669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:43.490057015Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:28:46.138992987Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=info ts=2026-04-20T12:28:46.502886384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:28:46.507195645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:28:47.751624201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:28:49.110285339Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:28:49.116028155Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:28:50.494053853Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-20T12:28:50.917810256Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:28:52.751586921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:54.849040951Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-20T12:28:56.240359702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:28:56.246943195Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:28:57.750850106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:28:57.750877741Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:28:57.750885342Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:28:57.750879779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:28:59.203355984Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=warn ts=2026-04-20T12:29:02.75082096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:03.55850875Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:29:07.751609676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:07.913245524Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
ts=2026-04-20T12:29:08.026644145Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:29:12.268655155Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=warn ts=2026-04-20T12:29:12.751046739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:15.24741322Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:29:16.622543863Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-20T12:29:17.750798474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:29:20.570275897Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:29:20.576831537Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:29:20.978245038Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-20T12:29:22.750833675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:25.123005731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:29:25.332743864Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-20T12:29:27.728499094Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:29:27.729551483Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:29:27.750909359Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:29:27.750935652Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:29:27.750945781Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:29:27.750941457Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:29.687942957Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=warn ts=2026-04-20T12:29:32.750969191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:34.042319907Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-04-20T12:29:35.904061682Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:29:35.90823399Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:29:37.751038684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:29:38.028884516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:29:38.397551663Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=warn ts=2026-04-20T12:29:42.751459896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:42.751536573Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=debug ts=2026-04-20T12:29:42.751552687Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=15
level=debug ts=2026-04-20T12:29:47.106476905Z caller=checkpoint.go:378 msg="writing series" size="349 kB"
level=warn ts=2026-04-20T12:29:47.750894245Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:47.958307603Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:29:49.500674877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:29:49.510164781Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:29:51.461803686Z caller=checkpoint.go:378 msg="writing series" size="377 kB"
level=warn ts=2026-04-20T12:29:52.751193775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:29:55.81662758Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=info ts=2026-04-20T12:29:57.751510353Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:29:57.751538508Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T12:29:57.75152498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:29:57.75154732Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:30:00.171386195Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
ts=2026-04-20T12:30:01.228163218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:51096"
level=warn ts=2026-04-20T12:30:02.751156879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:04.526193409Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:30:07.751633859Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:30:08.030836504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:30:08.880624263Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-20T12:30:08.880649921Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-20T12:30:12.751563573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:30:13.141251033Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:30:13.147535475Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:30:13.236280418Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=debug ts=2026-04-20T12:30:15.879222889Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:30:17.590188553Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:30:17.751720448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:18.329191183Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:30:20.344958398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:30:20.349206286Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:30:21.946058581Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=warn ts=2026-04-20T12:30:22.751645326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:26.300108286Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-20T12:30:27.728677698Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:30:27.7297398Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:30:27.729942475Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T12:30:27.729979956Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974097 n=1
level=debug ts=2026-04-20T12:30:27.730020975Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974097 n=1
level=info ts=2026-04-20T12:30:27.75112799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:30:27.751132613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:27.75114187Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:30:27.751150199Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:30:30.592285021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:30:30.600014482Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:30:30.65621171Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-20T12:30:30.656225325Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=5
ts=2026-04-20T12:30:31.23064404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:53512"
level=warn ts=2026-04-20T12:30:32.75154193Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:35.009791852Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-20T12:30:37.751183919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:30:38.033068764Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:30:39.123535028Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38328"
level=debug ts=2026-04-20T12:30:39.3652066Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=debug ts=2026-04-20T12:30:42.217410832Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:30:42.751547103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:43.71949065Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=warn ts=2026-04-20T12:30:47.751525157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:48.074010053Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=debug ts=2026-04-20T12:30:52.429331581Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=warn ts=2026-04-20T12:30:52.751299639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:30:56.7839822Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=debug ts=2026-04-20T12:30:57.003458883Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:30:57.751243065Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:30:57.751268959Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:30:57.751277266Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:30:57.751274825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:01.13961857Z caller=checkpoint.go:378 msg="writing series" size="466 kB"
level=warn ts=2026-04-20T12:31:02.751792761Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:05.49441431Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-20T12:31:07.750830648Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:31:08.034097534Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=info ts=2026-04-20T12:31:08.734979801Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:31:08.742883313Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-20T12:31:09.124687198Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:52256"
level=debug ts=2026-04-20T12:31:09.848287467Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-20T12:31:09.848319138Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-20T12:31:12.75149584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:12.897874323Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:31:12.898016728Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:31:12.907794497Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:31:14.204166902Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-20T12:31:16.653545476Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:31:16.65818154Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:31:17.751314642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:18.55882784Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=warn ts=2026-04-20T12:31:22.751096265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:22.913216472Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-20T12:31:25.779462882Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:31:25.787496847Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:31:27.268504392Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=debug ts=2026-04-20T12:31:27.728699994Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:31:27.729780573Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:31:27.751150499Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:31:27.751171244Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T12:31:27.751166804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:31:27.751181507Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:31:31.623171789Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=info ts=2026-04-20T12:31:32.011334208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:31:32.019498545Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:31:32.750932797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:35.977481621Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:31:37.751333595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:31:38.036086453Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:31:40.333264017Z caller=checkpoint.go:378 msg="writing series" size="395 kB"
level=warn ts=2026-04-20T12:31:42.751800135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:44.687634609Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-20T12:31:45.448535365Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:31:45.450286063Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:31:47.7508988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:48.393590723Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:31:49.04188551Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=info ts=2026-04-20T12:31:51.271063389Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:31:51.276922442Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:31:52.75105557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:53.396680725Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-20T12:31:54.570890685Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:31:54.578956626Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:31:57.751642046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:31:57.751651898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:31:57.751665656Z caller=checkpoint.go:390 msg="flushing series" totalSize="785 kB" series=10
level=debug ts=2026-04-20T12:31:57.751674286Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:31:57.7516853Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:31:57.84619234Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-20T12:31:57.847810264Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.09643846s
level=warn ts=2026-04-20T12:32:02.751465732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:32:05.627570231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:32:05.634382275Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:32:07.751025565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:32:08.037848297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:32:12.685649305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:32:12.750847372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:32:17.751318997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:32:20.215674376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:32:20.224348776Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:32:22.750889366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:32:26.034369699Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:59736"
level=info ts=2026-04-20T12:32:27.728883347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:32:27.729310562Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:32:27.750568816Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:32:27.750697857Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-20T12:32:27.750790168Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:32:27.75081083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:32:27.750819717Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:32:27.750830944Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:32:27.750930275Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
ts=2026-04-20T12:32:31.252076878Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:34466"
level=debug ts=2026-04-20T12:32:31.842071766Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=debug ts=2026-04-20T12:32:32.701946932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:32:32.751675499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:32:35.933569616Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-20T12:32:37.751342547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:32:38.039351201Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
ts=2026-04-20T12:32:39.128426095Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:56620"
level=info ts=2026-04-20T12:32:39.699094925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:32:39.706162495Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:32:40.024287832Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-04-20T12:32:42.751297733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:32:44.115289452Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-20T12:32:44.84786641Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:32:47.751647739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:32:48.205744991Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=info ts=2026-04-20T12:32:48.364470856Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:32:48.370052182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:32:52.296573744Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:32:52.751031128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:32:56.035421725Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:43850"
level=debug ts=2026-04-20T12:32:56.38826722Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:32:57.750854469Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:32:57.750884534Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:32:57.750895086Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:32:57.750887427Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:32:59.882126097Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:32:59.889303574Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:32:59.889326179Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:32:59.889348984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:32:59.892872239Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:33:00.478546153Z caller=checkpoint.go:378 msg="writing series" size="2.4 kB"
level=warn ts=2026-04-20T12:33:02.751069699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:04.569487855Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=warn ts=2026-04-20T12:33:07.751599576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:33:08.040632646Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:33:08.660199023Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:33:12.751603681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:12.751604794Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=info ts=2026-04-20T12:33:14.499685941Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:33:14.508650999Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:33:16.842026585Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:33:17.751291057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:19.024991482Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:33:19.332951996Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:33:19.339598961Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:33:19.339619547Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:33:19.339643201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:33:19.346522314Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:33:20.679211131Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:33:20.933424302Z caller=checkpoint.go:378 msg="writing series" size="508 kB"
level=debug ts=2026-04-20T12:33:20.933445959Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=warn ts=2026-04-20T12:33:22.751592147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:25.024316335Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
ts=2026-04-20T12:33:26.036966959Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38328"
level=info ts=2026-04-20T12:33:27.728953571Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:33:27.728969777Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:33:27.751404566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:33:27.751422217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:33:27.751429928Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:33:27.75144665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:33:28.617660309Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:33:28.6242675Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:33:29.119030811Z caller=checkpoint.go:378 msg="writing series" size="4.7 MB"
level=debug ts=2026-04-20T12:33:29.119050356Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.7 MB" series=2
level=info ts=2026-04-20T12:33:30.026799139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:33:30.033900905Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:33:32.750947235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:33.206112777Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=debug ts=2026-04-20T12:33:37.296702234Z caller=checkpoint.go:378 msg="writing series" size="161 kB"
level=warn ts=2026-04-20T12:33:37.7510636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:33:38.04201629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:33:41.387628919Z caller=checkpoint.go:378 msg="writing series" size="329 kB"
level=warn ts=2026-04-20T12:33:42.751686783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:45.484010687Z caller=checkpoint.go:378 msg="writing series" size="7.5 MB"
level=debug ts=2026-04-20T12:33:45.484036628Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.1 MB" series=4
level=info ts=2026-04-20T12:33:47.301643179Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:33:47.310003677Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:33:47.750787017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:49.569976421Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-04-20T12:33:51.004518245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:33:51.00832897Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:33:52.75094413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:33:53.660091488Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-04-20T12:33:55.235287342Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:33:55.242446561Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:33:55.383873237Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:33:56.038157418Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48100"
level=warn ts=2026-04-20T12:33:57.751016357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:33:57.751024036Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:33:57.751017319Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-20T12:33:57.751043324Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:33:57.751052252Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:34:01.842169951Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T12:34:02.751142936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:05.932901267Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-04-20T12:34:07.751510765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:34:08.043244924Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:34:08.95761388Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:34:10.024205067Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:34:12.751319892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:14.115023355Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=info ts=2026-04-20T12:34:17.58581525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:34:17.594803939Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:34:17.751510397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:18.206480423Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=debug ts=2026-04-20T12:34:22.296733894Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-20T12:34:22.75130161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:26.387282809Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=debug ts=2026-04-20T12:34:27.728648641Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:34:27.729721064Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:34:27.751073111Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:34:27.751098Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:34:27.751105564Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:34:27.751128407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:29.461622153Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:34:30.478479429Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-04-20T12:34:31.266289303Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:56456"
level=warn ts=2026-04-20T12:34:32.751588391Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:34:34.019197726Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:34:34.023335425Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:34:34.569224235Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=warn ts=2026-04-20T12:34:37.751455121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:34:38.044397504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=info ts=2026-04-20T12:34:38.230940578Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:34:38.235775854Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:34:38.660923698Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-20T12:34:42.751576562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:42.751638518Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=debug ts=2026-04-20T12:34:42.751654594Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-04-20T12:34:44.797173555Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:34:46.842666484Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=warn ts=2026-04-20T12:34:47.751202069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:50.933704321Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=warn ts=2026-04-20T12:34:52.751207705Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:34:53.808330842Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:34:53.816446334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:34:55.024023855Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
level=info ts=2026-04-20T12:34:57.751557995Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:34:57.751582491Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:34:57.751590728Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:34:57.751607945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:34:59.114811052Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T12:35:02.751214666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:03.083084835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:35:03.20583896Z caller=checkpoint.go:378 msg="writing series" size="215 kB"
level=info ts=2026-04-20T12:35:07.174087268Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:35:07.183772836Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:35:07.296995586Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:35:07.751441334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:35:08.046582584Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:35:11.387585898Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-20T12:35:12.751156514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:15.478659022Z caller=checkpoint.go:378 msg="writing series" size="255 kB"
level=warn ts=2026-04-20T12:35:17.751431694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:19.569470449Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-04-20T12:35:19.569495833Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-20T12:35:21.817872695Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:35:21.824237231Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:35:22.750925545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:23.660232934Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=debug ts=2026-04-20T12:35:26.235266936Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:35:26.236866408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:35:27.728900157Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:35:27.728938661Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:35:27.751362959Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:35:27.751382401Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:35:27.751390445Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:35:27.751397087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:27.751496922Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=debug ts=2026-04-20T12:35:31.843185102Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-20T12:35:31.843210094Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=3
level=warn ts=2026-04-20T12:35:32.751579765Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:35.933639239Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=info ts=2026-04-20T12:35:36.3886005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:35:36.395286452Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:35:36.395306499Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:35:36.395327591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:35:36.399662332Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:35:37.162933521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:35:37.751435204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:35:38.047912844Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:35:40.024000705Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=warn ts=2026-04-20T12:35:42.750837266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:44.115591102Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=warn ts=2026-04-20T12:35:47.751341182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:48.206397989Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:35:48.28559457Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:35:48.292565554Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-20T12:35:50.791301168Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:35:50.798015184Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:35:52.297323277Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-20T12:35:52.751672776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:35:56.388067209Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-04-20T12:35:57.751390534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:35:57.751416094Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:35:57.751424405Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:35:57.751449381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:00.478926413Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=debug ts=2026-04-20T12:36:00.478955458Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-20T12:36:01.854806382Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:36:01.864841045Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:36:02.751164745Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:04.569838277Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=warn ts=2026-04-20T12:36:07.751672806Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:36:08.049114139Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:36:08.660861208Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-20T12:36:08.799785885Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:36:11.792272265Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:36:11.797970231Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:36:12.75125608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:12.751354632Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-20T12:36:16.842417121Z caller=checkpoint.go:378 msg="writing series" size="82 kB"
level=warn ts=2026-04-20T12:36:17.750795684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:36:19.713316228Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:36:19.716789831Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:36:20.93286149Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=debug ts=2026-04-20T12:36:21.405710521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:36:22.75163898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:25.024113051Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=debug ts=2026-04-20T12:36:27.728752392Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:36:27.729786584Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:36:27.751204281Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:36:27.751223421Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:36:27.751231485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:36:27.75127701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:29.115617658Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-04-20T12:36:32.751718329Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:33.20617085Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-20T12:36:37.297190604Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:36:37.751582967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:36:38.050753724Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:36:41.3881535Z caller=checkpoint.go:378 msg="writing series" size="522 kB"
level=debug ts=2026-04-20T12:36:41.388176304Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
level=warn ts=2026-04-20T12:36:42.751572527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:43.512337512Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:36:45.479005124Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-20T12:36:47.751773147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:36:48.402836366Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:36:48.411869878Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:36:49.569504055Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-20T12:36:52.751207141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:36:53.660382913Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=debug ts=2026-04-20T12:36:57.751217502Z caller=checkpoint.go:390 msg="flushing series" totalSize="14 kB" series=3
level=warn ts=2026-04-20T12:36:57.751225592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:36:57.751234026Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:36:57.751257865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:36:57.751268792Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:36:57.75477829Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-20T12:36:57.758142586Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007302597s
level=info ts=2026-04-20T12:36:59.420042331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:36:59.429212313Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:37:01.823331698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:37:01.828566267Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:37:02.751720144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:37:07.750889652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:37:08.051695967Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:37:09.148962835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:53468"
level=warn ts=2026-04-20T12:37:12.751477517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:13.085929912Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:37:14.776359113Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:37:17.751431707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:37:18.57202257Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:37:18.58016288Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:37:22.751696283Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:37:27.728999348Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:37:27.72901817Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:37:27.750444399Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:37:27.750573182Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-20T12:37:27.750797498Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:37:27.750819374Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:37:27.750835963Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:37:27.750854029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:27.750869105Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-20T12:37:31.842342347Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-20T12:37:32.750819696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:37:35.187173423Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:37:35.194056865Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:37:35.932820544Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=warn ts=2026-04-20T12:37:37.751268117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:37:38.053261009Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
ts=2026-04-20T12:37:39.150871613Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:46456"
level=debug ts=2026-04-20T12:37:40.023542406Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=warn ts=2026-04-20T12:37:42.750905184Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:44.114829025Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T12:37:47.75123545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:48.206350598Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=debug ts=2026-04-20T12:37:49.107415222Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:37:50.646310082Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:37:50.654410143Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:37:50.654434651Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:37:50.654460667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:37:50.662249458Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:37:51.262874064Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:37:52.296776731Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:37:52.751547892Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:56.387389145Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:37:57.75133674Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:37:57.751342458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:37:57.75136924Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:37:57.751376992Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:37:58.95719752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:37:58.962243606Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:38:00.478330573Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-20T12:38:02.751463884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:04.570055914Z caller=checkpoint.go:378 msg="writing series" size="237 kB"
level=info ts=2026-04-20T12:38:06.159576552Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.8 MB" avg_comp="2.9 MB" total_uncomp="48 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:38:06.166825815Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:38:07.75156706Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:38:08.05491552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:38:08.660923037Z caller=checkpoint.go:378 msg="writing series" size="814 B"
ts=2026-04-20T12:38:09.154183679Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:38814"
level=warn ts=2026-04-20T12:38:12.751159379Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:12.751295289Z caller=checkpoint.go:378 msg="writing series" size="312 kB"
level=debug ts=2026-04-20T12:38:12.751311659Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=12
level=info ts=2026-04-20T12:38:14.019953374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:38:14.023569025Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:38:16.842173218Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-20T12:38:17.473125949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:38:17.480931649Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:38:17.750857798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:18.876308045Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:38:20.933319199Z caller=checkpoint.go:378 msg="writing series" size="533 kB"
level=warn ts=2026-04-20T12:38:22.751423608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:25.023747619Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
ts=2026-04-20T12:38:26.051415325Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:43396"
level=info ts=2026-04-20T12:38:27.729002344Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:38:27.729451452Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:38:27.750924525Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:38:27.750941658Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:38:27.750950088Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:38:27.751015981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:29.124926617Z caller=checkpoint.go:378 msg="writing series" size="9.0 MB"
level=debug ts=2026-04-20T12:38:29.124949865Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.5 MB" series=4
level=warn ts=2026-04-20T12:38:32.751231257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:33.206297739Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=debug ts=2026-04-20T12:38:37.296916169Z caller=checkpoint.go:378 msg="writing series" size="214 kB"
level=warn ts=2026-04-20T12:38:37.751242264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:38:37.894342626Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:38:37.919217148Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T12:38:38.056386419Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:38:41.387628588Z caller=checkpoint.go:378 msg="writing series" size="327 kB"
level=warn ts=2026-04-20T12:38:42.751594079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:43.852455021Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:38:45.482218834Z caller=checkpoint.go:378 msg="writing series" size="7.9 MB"
level=debug ts=2026-04-20T12:38:45.482241968Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.6 MB" series=4
level=info ts=2026-04-20T12:38:46.979367594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:38:46.984592246Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:38:47.751523287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:49.569426879Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-20T12:38:51.919022243Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:38:52.751572997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:38:53.660683994Z caller=checkpoint.go:378 msg="writing series" size="469 kB"
level=info ts=2026-04-20T12:38:57.751328408Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:38:57.751354788Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:38:57.751364637Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:38:57.751403044Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-20T12:38:57.751407474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:01.841736405Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T12:39:02.750810482Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:39:05.643137957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:39:05.647707626Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:39:05.874399843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:39:05.880922702Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:39:05.9335151Z caller=checkpoint.go:378 msg="writing series" size="521 kB"
level=debug ts=2026-04-20T12:39:05.933534765Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-20T12:39:07.751327773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:39:08.058070071Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:39:10.023847564Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-20T12:39:11.173577873Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:39:12.750913042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:14.1152483Z caller=checkpoint.go:378 msg="writing series" size="268 kB"
level=warn ts=2026-04-20T12:39:17.751630088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:18.205652217Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=debug ts=2026-04-20T12:39:22.296376459Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-20T12:39:22.751405446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:26.387742066Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-04-20T12:39:27.728882287Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:39:27.72932571Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:39:27.750749146Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:39:27.750786512Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:39:27.750794456Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:39:27.750797685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:39:28.841263535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:39:28.850461579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:39:30.479060485Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=warn ts=2026-04-20T12:39:32.751439496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:34.569253233Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=debug ts=2026-04-20T12:39:36.563800028Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:39:37.352935191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:39:37.358051374Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:39:37.751038972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:39:38.06051537Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:39:38.660509756Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-20T12:39:42.75172517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:42.751794327Z caller=checkpoint.go:378 msg="writing series" size="92 kB"
level=debug ts=2026-04-20T12:39:43.577946712Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:39:43.579574037Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:39:46.150089209Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:39:46.156584111Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:39:46.156603974Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:39:46.15662694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:39:46.162799114Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:39:46.84273923Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=warn ts=2026-04-20T12:39:47.750978681Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:50.933157265Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-20T12:39:50.933180135Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-04-20T12:39:52.751686505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:55.024048499Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
ts=2026-04-20T12:39:56.058411868Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:57858"
level=info ts=2026-04-20T12:39:57.75132991Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:39:57.751354295Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:39:57.751362538Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:39:57.751379997Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:39:59.115049357Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T12:40:02.751533751Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:03.206220561Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=info ts=2026-04-20T12:40:04.491982751Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:40:04.498320615Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:40:04.498341052Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:40:04.498362709Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:40:04.501615403Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:40:05.519754488Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.1 MB" avg_comp="2.6 MB" total_uncomp="43 MB" avg_uncomp="22 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:40:05.526193725Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:40:07.296667647Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:40:07.751173977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:40:08.062871607Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:40:09.160365858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58884"
level=debug ts=2026-04-20T12:40:10.805100179Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:40:11.38783212Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-20T12:40:12.75093384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:15.47904508Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-04-20T12:40:17.751240877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:19.569921707Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:40:22.751015499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:40:23.400264394Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:40:23.405316225Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:40:23.659970675Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=info ts=2026-04-20T12:40:26.798584197Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:40:26.803185629Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:40:27.728465935Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:40:27.729512322Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:40:27.751017494Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:40:27.751037084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:40:27.751045765Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:40:27.7510467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:27.751120065Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-04-20T12:40:27.84136553Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:40:27.848118576Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:40:28.563101425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:40:28.569587737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:40:31.84316044Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-20T12:40:31.843184229Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.8 MB" series=10
level=debug ts=2026-04-20T12:40:32.275134633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:40:32.751162882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:35.933359375Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-20T12:40:37.751245063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:40:38.064804625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:40:39.161648926Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58936"
level=debug ts=2026-04-20T12:40:40.023662414Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-04-20T12:40:40.565013803Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:40:42.751681686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:44.115482085Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=warn ts=2026-04-20T12:40:47.751400117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:48.205827633Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:40:51.063582549Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:40:51.069976662Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:40:52.297211597Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:40:52.751418756Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:40:56.388031236Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=info ts=2026-04-20T12:40:57.750870175Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:40:57.750898344Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:40:57.750909367Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:40:57.750922727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:00.479026734Z caller=checkpoint.go:378 msg="writing series" size="430 kB"
level=warn ts=2026-04-20T12:41:02.751558613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:04.569986112Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=debug ts=2026-04-20T12:41:04.570010559Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-20T12:41:07.751205429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:41:08.065658972Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:41:08.147485292Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:41:08.66013971Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=debug ts=2026-04-20T12:41:10.092605919Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:41:12.751008056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:12.751047825Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-20T12:41:16.842262375Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-04-20T12:41:17.75133796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:41:18.824287591Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.7 MB" avg_comp="2.9 MB" total_uncomp="73 MB" avg_uncomp="24 MB" full=3 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:41:18.834632892Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:41:19.550715439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:41:19.555790339Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:41:20.93270669Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=info ts=2026-04-20T12:41:21.930470592Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:41:21.938386315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:41:22.750852551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:25.023595323Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=info ts=2026-04-20T12:41:27.728890009Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:41:27.729173172Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:41:27.751692485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:41:27.751707714Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:41:27.75171552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:41:27.751726035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:29.114600327Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
ts=2026-04-20T12:41:31.307018798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:43936"
level=info ts=2026-04-20T12:41:32.318115506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:41:32.324363416Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:41:32.750967902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:33.206244838Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=debug ts=2026-04-20T12:41:37.296593566Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:41:37.750947398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:41:38.067652409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:41:39.164131287Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:44452"
level=debug ts=2026-04-20T12:41:41.388324283Z caller=checkpoint.go:378 msg="writing series" size="387 kB"
level=debug ts=2026-04-20T12:41:41.998878066Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:41:41.998943383Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="541 kB" avg_comp="541 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:41:42.003836752Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:41:42.751579405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:45.478992254Z caller=checkpoint.go:378 msg="writing series" size="8.2 kB"
level=warn ts=2026-04-20T12:41:47.751640491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:49.569667779Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=warn ts=2026-04-20T12:41:52.751329141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:41:53.66004267Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-20T12:41:56.066464349Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:52442"
level=info ts=2026-04-20T12:41:57.750774542Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:41:57.750790124Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=12
level=debug ts=2026-04-20T12:41:57.75081071Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:41:57.750820313Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:41:57.750818654Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:41:57.758910547Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-20T12:41:57.762313697Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.011575107s
level=info ts=2026-04-20T12:41:58.757517498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:41:58.765882541Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:42:02.751409531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:42:05.209907816Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:42:05.963002192Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:42:05.96972238Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:42:07.751586626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:42:08.069994909Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:42:08.371145639Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:42:12.750945678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:42:13.220229659Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:42:13.227918165Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:42:17.527933441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:42:17.534346575Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:42:17.534367126Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:42:17.534393112Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:42:17.539243317Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:42:17.751682312Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:42:22.750976741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:42:26.067456803Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:38176"
level=info ts=2026-04-20T12:42:26.580986581Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:42:26.585484691Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:42:27.728828342Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:42:27.729883619Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:42:27.75030381Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:42:27.750438266Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-20T12:42:27.750679707Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-04-20T12:42:27.750780587Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:42:27.75081398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:42:27.750821588Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:42:27.750813539Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:42:31.315210521Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:40466"
level=debug ts=2026-04-20T12:42:31.781692809Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-20T12:42:31.781716222Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=2
level=info ts=2026-04-20T12:42:31.801067525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:42:31.806266838Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:42:32.751591431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:42:35.663181574Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:42:35.668888311Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:42:35.810546523Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=warn ts=2026-04-20T12:42:37.750812606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:42:38.07157964Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:42:39.84122389Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-20T12:42:42.750936883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:42:43.511884585Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:42:43.870946106Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-20T12:42:47.751266932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:42:47.900684428Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-20T12:42:51.930047814Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:42:52.751457107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:42:55.643858167Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:42:55.959729115Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-20T12:42:56.069022953Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:46392"
level=info ts=2026-04-20T12:42:57.474125215Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:42:57.480739923Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:42:57.750788921Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:42:57.750806344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:42:57.750818233Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:42:57.750826851Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:42:59.989881057Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=info ts=2026-04-20T12:43:01.244336815Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:43:01.252221493Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:43:02.750943869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:04.019668727Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=info ts=2026-04-20T12:43:06.289156651Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:43:06.29599879Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:43:07.751698704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:08.049693136Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=debug ts=2026-04-20T12:43:08.0497142Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
ts=2026-04-20T12:43:08.073920374Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:43:12.079286338Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:43:12.75087491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:16.10956063Z caller=checkpoint.go:378 msg="writing series" size="550 kB"
level=info ts=2026-04-20T12:43:16.709176577Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:43:16.715611434Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:43:17.751732655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:17.90388445Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:43:20.139620032Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-20T12:43:21.142469343Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:43:21.146731332Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:43:22.751589709Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:24.169298412Z caller=checkpoint.go:378 msg="writing series" size="786 kB"
level=debug ts=2026-04-20T12:43:24.16931968Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=4
level=info ts=2026-04-20T12:43:27.728947522Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:43:27.729377755Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:43:27.750887485Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:43:27.750906429Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:43:27.750916077Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:43:27.750917418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:28.199378098Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=debug ts=2026-04-20T12:43:32.235292876Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-20T12:43:32.235318018Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=2
level=warn ts=2026-04-20T12:43:32.751740795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:34.478721012Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:43:34.480848414Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:43:36.258410133Z caller=checkpoint.go:378 msg="writing series" size="141 kB"
level=warn ts=2026-04-20T12:43:37.75096857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:43:38.076268288Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:43:40.288439193Z caller=checkpoint.go:378 msg="writing series" size="61 kB"
level=warn ts=2026-04-20T12:43:42.751797256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:44.31868996Z caller=checkpoint.go:378 msg="writing series" size="390 kB"
level=info ts=2026-04-20T12:43:46.870877791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:43:46.879549934Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:43:47.433207059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:43:47.439963076Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:43:47.750812279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:48.352685566Z caller=checkpoint.go:378 msg="writing series" size="8.6 MB"
level=debug ts=2026-04-20T12:43:48.352706379Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=4
level=debug ts=2026-04-20T12:43:52.378610582Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=debug ts=2026-04-20T12:43:52.441911339Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:43:52.750857417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:43:56.407774517Z caller=checkpoint.go:378 msg="writing series" size="582 kB"
level=info ts=2026-04-20T12:43:57.750950576Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:43:57.750985688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:43:57.750998085Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:43:57.750999275Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:00.437593193Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-20T12:44:01.323360047Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:51214"
level=warn ts=2026-04-20T12:44:02.751025684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:04.467543989Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-20T12:44:04.665212157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:44:04.677638318Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:44:07.751381903Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:44:08.078430603Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:44:08.498147882Z caller=checkpoint.go:378 msg="writing series" size="723 kB"
level=debug ts=2026-04-20T12:44:08.49817148Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=5
ts=2026-04-20T12:44:09.174180858Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:58224"
level=info ts=2026-04-20T12:44:09.494263053Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:44:09.498919867Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-20T12:44:10.355595039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:44:10.359989535Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:44:12.52696562Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:44:12.751308964Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:16.556888269Z caller=checkpoint.go:378 msg="writing series" size="250 kB"
level=warn ts=2026-04-20T12:44:17.750859486Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:44:18.066567885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:44:18.066601088Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-20T12:44:18.078090981Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:44:20.587269006Z caller=checkpoint.go:378 msg="writing series" size="181 kB"
level=warn ts=2026-04-20T12:44:22.751011034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:24.616784969Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=debug ts=2026-04-20T12:44:27.728511386Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:44:27.729574592Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:44:27.75109913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:44:27.751105955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:27.751114896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:44:27.751124682Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:44:28.646177007Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=debug ts=2026-04-20T12:44:31.105962101Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:44:32.676179823Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-20T12:44:32.751482581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:44:33.075809149Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:44:33.083723113Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:44:33.083743856Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:44:33.083785585Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:44:33.08995268Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:44:35.767077828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:44:35.77464341Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:44:36.706643548Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=warn ts=2026-04-20T12:44:37.751660994Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:44:38.079848311Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:44:39.174866007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:39790"
level=debug ts=2026-04-20T12:44:40.736323916Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-20T12:44:42.751726276Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:44.76571602Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=info ts=2026-04-20T12:44:44.990294867Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:44:44.996847315Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:44:47.365307305Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:44:47.751328452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:48.79615924Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-20T12:44:52.75126477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:44:52.825879851Z caller=checkpoint.go:378 msg="writing series" size="444 kB"
level=debug ts=2026-04-20T12:44:52.825896192Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=11
level=info ts=2026-04-20T12:44:52.980995024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:44:52.992616985Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T12:44:56.075384081Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:54386"
level=debug ts=2026-04-20T12:44:56.856074868Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=info ts=2026-04-20T12:44:57.751336802Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:44:57.751366111Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:44:57.751376944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:44:57.751389579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:45:00.386176365Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:45:00.390063671Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:45:00.885556614Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T12:45:02.751191173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:45:03.903581102Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:45:03.912042638Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=info ts=2026-04-20T12:45:04.202404388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:45:04.207142Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:45:04.915279933Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=info ts=2026-04-20T12:45:06.330165537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:45:06.335712874Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:45:07.751689477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:45:08.081196731Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:45:08.945137752Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-20T12:45:09.176189625Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:41408"
level=debug ts=2026-04-20T12:45:11.434742363Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:45:12.750953683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:12.974852799Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-20T12:45:17.005402937Z caller=checkpoint.go:378 msg="writing series" size="157 kB"
level=warn ts=2026-04-20T12:45:17.751409181Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:45:19.872578544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:45:19.882374322Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:45:21.0345875Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=debug ts=2026-04-20T12:45:22.333854236Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:45:22.751706246Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:25.064951345Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=info ts=2026-04-20T12:45:27.728880998Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:45:27.729152678Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=debug ts=2026-04-20T12:45:27.730410054Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T12:45:27.730446792Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974098 n=1
level=debug ts=2026-04-20T12:45:27.730493925Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974098 n=1
level=info ts=2026-04-20T12:45:27.751683578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:45:27.751699765Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:45:27.751707208Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:45:27.751712937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:29.094892894Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=debug ts=2026-04-20T12:45:29.094914624Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
ts=2026-04-20T12:45:31.334173077Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42908"
level=warn ts=2026-04-20T12:45:32.751376841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:33.124746767Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-20T12:45:33.124782909Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.8 MB" series=1
level=debug ts=2026-04-20T12:45:37.153941938Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=warn ts=2026-04-20T12:45:37.751510115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:45:38.082326418Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:45:38.320570516Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:45:38.741918956Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:45:38.749826761Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:45:41.183911793Z caller=checkpoint.go:378 msg="writing series" size="310 kB"
level=warn ts=2026-04-20T12:45:42.751165313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:45:43.904862942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:45:43.908906094Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:45:45.213881689Z caller=checkpoint.go:378 msg="writing series" size="345 kB"
level=info ts=2026-04-20T12:45:47.709786872Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:45:47.716709921Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T12:45:47.750913201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:49.243787506Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:45:52.751565821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:45:53.273559165Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-04-20T12:45:57.30323286Z caller=checkpoint.go:378 msg="writing series" size="35 kB"
level=info ts=2026-04-20T12:45:57.750859945Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:45:57.750889445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:45:57.750898335Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:45:57.750896088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:45:59.299611015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:45:59.306688402Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:46:01.333900997Z caller=checkpoint.go:378 msg="writing series" size="537 kB"
level=debug ts=2026-04-20T12:46:01.333926262Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=debug ts=2026-04-20T12:46:02.49798225Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:46:02.7515355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:03.665521961Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:46:05.36301337Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=info ts=2026-04-20T12:46:05.959792139Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:46:05.9638013Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:46:07.751334752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:46:08.084246601Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:46:09.178329842Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:44862"
level=debug ts=2026-04-20T12:46:09.392489005Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-20T12:46:11.30765495Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="12 MB" avg_comp="2.9 MB" total_uncomp="97 MB" avg_uncomp="24 MB" full=4 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:46:11.315621715Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:46:12.751588458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:13.422631268Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-20T12:46:17.452961513Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-04-20T12:46:17.750979479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:21.482954294Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T12:46:22.751189254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:46:25.433301773Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:46:25.439464386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:46:25.439486566Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:46:25.439511287Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:46:25.442960646Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:46:25.512108289Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-20T12:46:26.799867705Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:46:26.806384617Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:46:26.806405898Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:46:26.80643121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:46:26.81228183Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:46:27.728997657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:46:27.728997612Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:46:27.751595371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:46:27.751628031Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:46:27.751640293Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:46:27.751671804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:29.542667856Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-20T12:46:32.751530511Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:33.572335182Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=info ts=2026-04-20T12:46:35.543161162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:46:35.552123086Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:46:36.349333063Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:46:37.23264605Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:46:37.236475398Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:46:37.601394779Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:46:37.750880063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:46:38.08663211Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:46:41.631816347Z caller=checkpoint.go:378 msg="writing series" size="515 kB"
level=debug ts=2026-04-20T12:46:41.631842188Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-04-20T12:46:42.376704504Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:46:42.383800985Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:46:42.75091633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:45.661671037Z caller=checkpoint.go:378 msg="writing series" size="9.3 kB"
level=warn ts=2026-04-20T12:46:47.751355116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:46:49.387870993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:46:49.391920679Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:46:49.691911447Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
level=debug ts=2026-04-20T12:46:51.374051421Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:46:52.75108532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:46:53.721824669Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-20T12:46:57.750876361Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:46:57.7508818Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 kB" series=3
level=debug ts=2026-04-20T12:46:57.750904001Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:46:57.750912332Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:46:57.750931417Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:46:57.756195413Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-20T12:46:57.757909744Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007332733s
level=info ts=2026-04-20T12:46:59.055070233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:46:59.059875647Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-20T12:47:01.601525262Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="122 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:47:01.608563058Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:47:02.751496678Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:47:07.751598854Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:47:08.087666574Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=info ts=2026-04-20T12:47:08.238797938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:47:08.246552435Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:47:08.811162079Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:47:12.751470394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:47:13.968886318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:47:13.976926935Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:47:17.751681814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:47:21.428962079Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:47:21.436288133Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T12:47:22.750957814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:26.49400731Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:47:26.495618243Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-20T12:47:27.72889257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:47:27.729127791Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:47:27.750549442Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:47:27.750676026Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-20T12:47:27.750727103Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:47:27.750752224Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:47:27.750778972Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:47:27.750813885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:27.750973518Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=debug ts=2026-04-20T12:47:31.783845162Z caller=checkpoint.go:378 msg="writing series" size="4.5 MB"
level=debug ts=2026-04-20T12:47:31.783870344Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.7 MB" series=2
level=warn ts=2026-04-20T12:47:32.751588839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:35.811694787Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=warn ts=2026-04-20T12:47:37.751391513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:47:38.08980315Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:47:39.841081627Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=debug ts=2026-04-20T12:47:41.726318645Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:47:42.751376918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:43.871167535Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-20T12:47:45.122058364Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:47:45.130477909Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:47:47.751453418Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:47.900887303Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=debug ts=2026-04-20T12:47:51.930076748Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:47:52.751164732Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:47:55.862589416Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:47:55.870202079Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:47:55.960516955Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:47:56.396781521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="122 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:47:56.403902712Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:47:57.751635327Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:47:57.751631884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:47:57.751658977Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:47:57.751667448Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:47:59.856752617Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:47:59.860798857Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:47:59.990311907Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-04-20T12:47:59.990329411Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
ts=2026-04-20T12:48:01.351817855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:44120"
level=info ts=2026-04-20T12:48:02.504240796Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:48:02.511348172Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T12:48:02.751320171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:48:03.313738781Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:48:03.320405548Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:48:04.02032529Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=debug ts=2026-04-20T12:48:07.206958516Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:48:07.751543798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:08.049883316Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
ts=2026-04-20T12:48:08.092534987Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:48:09.757657378Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:48:12.079696068Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:48:12.750877833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:16.110494641Z caller=checkpoint.go:378 msg="writing series" size="550 kB"
level=warn ts=2026-04-20T12:48:17.750855178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:20.139525018Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:48:22.751783634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:24.169969929Z caller=checkpoint.go:378 msg="writing series" size="827 kB"
level=debug ts=2026-04-20T12:48:24.169990402Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=6
level=info ts=2026-04-20T12:48:27.729481251Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:48:27.729970795Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:48:27.751431839Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:48:27.751447317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:27.751454278Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:48:27.75146399Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:48:28.199251953Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-20T12:48:31.741604331Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:48:31.749196216Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:48:31.74921739Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:48:31.749241862Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:48:31.753686353Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:48:32.243010891Z caller=checkpoint.go:378 msg="writing series" size="16 MB"
level=debug ts=2026-04-20T12:48:32.243034368Z caller=checkpoint.go:390 msg="flushing series" totalSize="16 MB" series=2
level=warn ts=2026-04-20T12:48:32.751580961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:35.235690871Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:48:36.258701669Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-20T12:48:37.750944229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:48:38.095491624Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:48:40.288518225Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-20T12:48:42.75160235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:48:43.376822283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:48:43.384100703Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-20T12:48:43.75126971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="122 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:48:43.758143241Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:48:44.319092289Z caller=checkpoint.go:378 msg="writing series" size="511 kB"
level=warn ts=2026-04-20T12:48:47.751334884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:48:47.839280638Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:48:47.843871688Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:48:48.356212228Z caller=checkpoint.go:378 msg="writing series" size="9.1 MB"
level=debug ts=2026-04-20T12:48:48.356243393Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.8 MB" series=4
level=info ts=2026-04-20T12:48:49.691691988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:48:49.69890675Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:48:52.378933597Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=info ts=2026-04-20T12:48:52.622635555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:48:52.626810328Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:48:52.751010433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:48:55.804865691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:48:55.811183962Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-20T12:48:56.091455118Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:45946"
level=debug ts=2026-04-20T12:48:56.407893765Z caller=checkpoint.go:378 msg="writing series" size="621 kB"
level=info ts=2026-04-20T12:48:57.751362096Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:48:57.751391636Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:48:57.751399666Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:48:57.751439229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:48:58.084022812Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:49:00.438538601Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-20T12:49:01.3576948Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:50768"
level=warn ts=2026-04-20T12:49:02.750871613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:04.468204987Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-20T12:49:04.516384521Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:49:07.75118823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:49:08.097915036Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:49:08.498315054Z caller=checkpoint.go:378 msg="writing series" size="804 kB"
level=debug ts=2026-04-20T12:49:08.498337919Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=5
level=debug ts=2026-04-20T12:49:12.527314699Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:49:12.751589414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:16.557724621Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=warn ts=2026-04-20T12:49:17.751177694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:20.587449181Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=info ts=2026-04-20T12:49:20.682617019Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:49:20.688074607Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T12:49:22.751198845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:24.617140442Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
ts=2026-04-20T12:49:26.09314405Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:60592"
level=info ts=2026-04-20T12:49:27.037208021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="122 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:49:27.044153096Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:49:27.728692506Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:49:27.729744217Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:49:27.751188281Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:49:27.751199942Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:27.751210195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:49:27.751220242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:49:28.647267376Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=debug ts=2026-04-20T12:49:32.676694824Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-20T12:49:32.75117958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:49:34.72367238Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=1 total_comp="3.0 MB" avg_comp="3.0 MB" total_uncomp="12 MB" avg_uncomp="12 MB" full=1 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:49:34.733077501Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:49:36.707100617Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
level=debug ts=2026-04-20T12:49:36.883730513Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:49:37.751265906Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:49:38.100244559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:49:40.149015371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:49:40.15337051Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:49:40.736206946Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-20T12:49:42.751040943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:49:42.796888473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:49:42.803322982Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:49:44.765950414Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
level=debug ts=2026-04-20T12:49:44.765975777Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=info ts=2026-04-20T12:49:46.442974518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:49:46.448243118Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:49:46.448261769Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:49:46.448284245Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:49:46.452446612Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:49:47.750849779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:48.796569705Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-04-20T12:49:52.751185471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:49:52.826651004Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=debug ts=2026-04-20T12:49:56.300894966Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:49:56.855787615Z caller=checkpoint.go:378 msg="writing series" size="389 kB"
level=info ts=2026-04-20T12:49:57.751303123Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:49:57.751330236Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:49:57.751338586Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:49:57.751382502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:50:00.590037918Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:50:00.596474577Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:50:00.886295426Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-20T12:50:02.292213986Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:50:02.751409965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:04.915348517Z caller=checkpoint.go:378 msg="writing series" size="380 kB"
level=debug ts=2026-04-20T12:50:04.915371941Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-20T12:50:07.750891572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:50:08.103487357Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:50:08.945628918Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-20T12:50:12.751363896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:12.975543976Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-20T12:50:13.748065341Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:50:13.756159109Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:50:17.005709063Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-20T12:50:17.750965559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:50:18.431628388Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=5 total_comp="14 MB" avg_comp="2.9 MB" total_uncomp="122 MB" avg_uncomp="24 MB" full=5 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:50:18.438469685Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 5, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:50:21.035334238Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:50:22.750822737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:50:24.953812219Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:50:24.962585352Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:50:24.962609338Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:50:24.962635231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:50:24.970115402Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:50:25.06459371Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=info ts=2026-04-20T12:50:27.728930742Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:50:27.729372825Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:50:27.750845434Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:50:27.750863832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:50:27.750871597Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:50:27.750874441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:29.094688093Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-04-20T12:50:29.563377273Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:50:30.80295314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:50:30.808079228Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:50:32.751574113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:33.126056321Z caller=checkpoint.go:378 msg="writing series" size="2.0 MB"
level=debug ts=2026-04-20T12:50:33.126077669Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.6 MB" series=7
level=info ts=2026-04-20T12:50:36.095618921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:50:36.103630888Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:50:37.154653401Z caller=checkpoint.go:378 msg="writing series" size="557 B"
level=info ts=2026-04-20T12:50:37.594769282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:50:37.598227208Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:50:37.751674103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:50:38.106503111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:50:38.704676376Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:50:38.711653029Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:50:41.184387084Z caller=checkpoint.go:378 msg="writing series" size="484 kB"
level=warn ts=2026-04-20T12:50:42.751568874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:45.214655668Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-20T12:50:47.750891919Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:49.244514316Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:50:52.751562525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:50:53.274048757Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-20T12:50:54.141806475Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:50:54.15031056Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:50:57.304243393Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-20T12:50:57.38471149Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:50:57.404804347Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:50:57.750856269Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:50:57.750885125Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:50:57.750894309Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:50:57.750911477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:01.333976648Z caller=checkpoint.go:378 msg="writing series" size="486 kB"
level=debug ts=2026-04-20T12:51:01.334002421Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=7
level=warn ts=2026-04-20T12:51:02.750947818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:51:04.605350417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:51:04.613603053Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:51:05.362942059Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-20T12:51:07.750880206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:51:08.108463183Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:51:09.393059483Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=info ts=2026-04-20T12:51:12.589959282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:51:12.594846946Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:51:12.751160226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:13.423498814Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-20T12:51:15.079720989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="146 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:51:15.087000056Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:51:15.08701989Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:51:15.087048632Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="146 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:51:15.0933991Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:51:17.453097421Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=warn ts=2026-04-20T12:51:17.751633691Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:51:20.233405211Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:51:20.237570036Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:51:21.483055905Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T12:51:22.751826384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:25.512919323Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-20T12:51:26.676207503Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:51:27.728920125Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:51:27.729266996Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:51:27.750876621Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:51:27.750893833Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T12:51:27.750888168Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:51:27.750901989Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:51:28.870954746Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:51:28.883013157Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:51:29.542872312Z caller=checkpoint.go:378 msg="writing series" size="211 kB"
level=info ts=2026-04-20T12:51:30.809452297Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="146 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:51:30.816458507Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:51:32.751559241Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:33.571987037Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-20T12:51:37.60247922Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:51:37.751713336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:51:38.110163433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:51:41.632700298Z caller=checkpoint.go:378 msg="writing series" size="615 kB"
level=debug ts=2026-04-20T12:51:41.632723304Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=10
level=warn ts=2026-04-20T12:51:42.751506149Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:42.968730096Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:51:42.97022286Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:51:45.661615691Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:51:47.751452973Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:51:48.245612525Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:51:48.249646932Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:51:49.691411993Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-20T12:51:52.670893194Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="146 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:51:52.679073184Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:51:52.751119479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:53.721435237Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-20T12:51:56.103615364Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:42212"
level=info ts=2026-04-20T12:51:57.750793581Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:51:57.750809364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:51:57.750824413Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:51:57.750835158Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:51:57.751893997Z caller=checkpoint.go:390 msg="flushing series" totalSize="17 kB" series=3
level=info ts=2026-04-20T12:51:57.755418254Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-20T12:51:57.756420377Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.005549615s
level=debug ts=2026-04-20T12:51:57.801114308Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:52:00.953193208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:52:00.962310595Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
ts=2026-04-20T12:52:01.375822089Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:33430"
level=warn ts=2026-04-20T12:52:02.751444401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:52:04.754237231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:52:04.758910195Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:52:07.750822578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:52:08.111511621Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
ts=2026-04-20T12:52:09.201075293Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:35356"
level=info ts=2026-04-20T12:52:12.145292446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:52:12.150443791Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:52:12.751394985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:52:17.751509554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:52:17.843813973Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:52:17.855446078Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:52:22.750959784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:52:22.88236173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:52:27.028055507Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:52:27.728703845Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:52:27.729747134Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:52:27.750252201Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:52:27.750388166Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-20T12:52:27.750645674Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=warn ts=2026-04-20T12:52:27.751268642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:52:27.751274064Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:52:27.751293172Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:52:27.751301701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-20T12:52:31.378328697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:38692"
level=debug ts=2026-04-20T12:52:31.786423793Z caller=checkpoint.go:378 msg="writing series" size="6.3 MB"
level=debug ts=2026-04-20T12:52:31.786449857Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.5 MB" series=2
level=warn ts=2026-04-20T12:52:32.750872484Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:52:33.477294542Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:52:33.482568248Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-20T12:52:33.549032752Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=6 total_comp="17 MB" avg_comp="2.9 MB" total_uncomp="146 MB" avg_uncomp="24 MB" full=6 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:52:33.55589197Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 6, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:52:35.811461296Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=warn ts=2026-04-20T12:52:37.750809748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:52:38.112791485Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:52:39.840670477Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-20T12:52:42.751506428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:52:43.870369874Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-20T12:52:45.308813537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:52:45.316665887Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:52:45.316687425Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:52:45.316712133Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:52:45.320083681Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:52:47.751339205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:52:47.902025579Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-20T12:52:49.916878334Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:52:51.563632901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=2 total_comp="6.0 MB" avg_comp="3.0 MB" total_uncomp="24 MB" avg_uncomp="12 MB" full=2 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:52:51.57126799Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:52:51.930409166Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:52:52.751321223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:52:55.960450438Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-20T12:52:57.751170166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:52:57.75119824Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:52:57.751208515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:52:57.751216615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:52:59.990430034Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-20T12:53:02.750890905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:53:03.105277344Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:53:03.11389018Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:53:03.11391024Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:53:03.11393562Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:53:03.120116538Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:53:04.019979269Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-20T12:53:05.128502505Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:53:05.134373255Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:53:07.75079373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:08.049414497Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
level=debug ts=2026-04-20T12:53:08.049435575Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-04-20T12:53:08.114437338Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:53:09.063799174Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:53:09.069203261Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:53:12.079649052Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:53:12.751372887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:16.109515894Z caller=checkpoint.go:378 msg="writing series" size="550 kB"
level=warn ts=2026-04-20T12:53:17.751661387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:18.322033948Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:53:18.818397757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:53:18.825339643Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:53:19.304700881Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:53:20.139528248Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:53:22.751801984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:24.169258081Z caller=checkpoint.go:378 msg="writing series" size="898 kB"
level=debug ts=2026-04-20T12:53:24.169288778Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=4
level=info ts=2026-04-20T12:53:25.124195477Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:53:25.128578099Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:53:27.728801067Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:53:27.729862028Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:53:27.751328314Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:53:27.751347649Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-20T12:53:27.751337996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:53:27.751355095Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-20T12:53:28.182096445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:53:28.185738697Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:53:28.198832846Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
ts=2026-04-20T12:53:31.384930989Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:42458"
level=debug ts=2026-04-20T12:53:32.263043862Z caller=checkpoint.go:378 msg="writing series" size="20 MB"
level=debug ts=2026-04-20T12:53:32.263069334Z caller=checkpoint.go:390 msg="flushing series" totalSize="20 MB" series=2
level=info ts=2026-04-20T12:53:32.702573891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:53:32.710224408Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:53:32.75124885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:36.258167173Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-20T12:53:37.750887662Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:53:38.117437976Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:53:40.288244595Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=warn ts=2026-04-20T12:53:42.751264811Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:43.31233102Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:53:44.318162125Z caller=checkpoint.go:378 msg="writing series" size="386 kB"
level=info ts=2026-04-20T12:53:47.251369441Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:53:47.261893596Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T12:53:47.751149622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:53:48.356201012Z caller=checkpoint.go:378 msg="writing series" size="9.6 MB"
level=debug ts=2026-04-20T12:53:48.356223897Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=4
level=info ts=2026-04-20T12:53:50.822121731Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:53:50.828671562Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:53:52.378411001Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=warn ts=2026-04-20T12:53:52.751220651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:53:54.828436415Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:53:54.83296559Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:53:55.374274278Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:53:56.408615048Z caller=checkpoint.go:378 msg="writing series" size="917 kB"
level=info ts=2026-04-20T12:53:57.751167942Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:53:57.751196497Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:53:57.751204156Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:53:57.751248984Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:00.437821593Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-20T12:54:02.751396148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:04.467728732Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T12:54:07.7517384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:54:08.118976864Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:54:08.498180773Z caller=checkpoint.go:378 msg="writing series" size="931 kB"
level=debug ts=2026-04-20T12:54:08.498205033Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.0 MB" series=5
level=debug ts=2026-04-20T12:54:09.783130282Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:54:12.526912613Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:54:12.751561496Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:54:14.346694026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:54:14.351688435Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:54:16.556846001Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=warn ts=2026-04-20T12:54:17.750812205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:54:18.440533811Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:54:18.4467458Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:54:20.506733445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:54:20.513546655Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:54:20.58697123Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=warn ts=2026-04-20T12:54:22.750877247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:24.616421078Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-20T12:54:26.114691251Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:48048"
level=debug ts=2026-04-20T12:54:27.728578831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:54:27.72964249Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:54:27.751152799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:54:27.75117069Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:54:27.751179421Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:54:27.751195646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:28.646525287Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
ts=2026-04-20T12:54:31.391168823Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:48944"
level=debug ts=2026-04-20T12:54:32.676169633Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-20T12:54:32.75155902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:36.706526543Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=warn ts=2026-04-20T12:54:37.751365931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:54:38.121454633Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:54:39.211037612Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:39968"
level=debug ts=2026-04-20T12:54:40.735893524Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-20T12:54:42.751191909Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:43.198534972Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:54:43.246797318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:54:43.254712298Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=info ts=2026-04-20T12:54:44.666234694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:54:44.672348173Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:54:44.765872171Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
level=debug ts=2026-04-20T12:54:46.644723844Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-20T12:54:47.751614977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:48.796196786Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=debug ts=2026-04-20T12:54:48.796219333Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=info ts=2026-04-20T12:54:52.524245062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:54:52.532083011Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:54:52.751307376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:54:52.825844537Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=debug ts=2026-04-20T12:54:56.855575436Z caller=checkpoint.go:378 msg="writing series" size="414 kB"
level=warn ts=2026-04-20T12:54:57.750883317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:54:57.75088484Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:54:57.750911398Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:54:57.75091918Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:55:00.885824714Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T12:55:02.751310343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:55:02.914606446Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:55:02.920672279Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:55:03.295380005Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:55:03.299711125Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:55:03.299730259Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:55:03.299753609Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:55:03.303238587Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:55:04.915877533Z caller=checkpoint.go:378 msg="writing series" size="290 kB"
level=debug ts=2026-04-20T12:55:04.915897048Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=4
level=debug ts=2026-04-20T12:55:07.668795773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:55:07.750970532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:55:08.12362507Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:55:08.945363237Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-20T12:55:11.699642015Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:55:11.706406894Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:55:12.751378411Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:12.975432738Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-20T12:55:17.005344067Z caller=checkpoint.go:378 msg="writing series" size="269 kB"
level=warn ts=2026-04-20T12:55:17.751348708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:21.034701707Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T12:55:22.75116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:55:23.275612294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:55:23.283253125Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:55:25.064686597Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
ts=2026-04-20T12:55:26.119068398Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:51896"
level=info ts=2026-04-20T12:55:27.72912069Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:55:27.729644589Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:55:27.751162248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:55:27.751193173Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:55:27.751200907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:55:27.751186126Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:29.094280472Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
ts=2026-04-20T12:55:31.397221102Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:56220"
level=warn ts=2026-04-20T12:55:32.751336652Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:33.12498477Z caller=checkpoint.go:378 msg="writing series" size="2.2 MB"
level=debug ts=2026-04-20T12:55:33.125005144Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=7
level=info ts=2026-04-20T12:55:36.539711691Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:55:36.546416356Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:55:37.154097199Z caller=checkpoint.go:378 msg="writing series" size="643 B"
level=warn ts=2026-04-20T12:55:37.751642177Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:55:38.126434768Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T12:55:38.667131374Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:55:38.673997638Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:55:39.073086694Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:55:39.080013728Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
ts=2026-04-20T12:55:39.213217838Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:35570"
level=debug ts=2026-04-20T12:55:39.485630891Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:55:41.183883745Z caller=checkpoint.go:378 msg="writing series" size="548 kB"
level=warn ts=2026-04-20T12:55:42.751517819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:55:43.36777723Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:55:43.371606413Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:55:43.491171271Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:55:43.492668879Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-20T12:55:45.21364185Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=warn ts=2026-04-20T12:55:47.751394917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:49.243157114Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:55:52.751788723Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:55:53.273694744Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=info ts=2026-04-20T12:55:54.862670998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:55:54.86871682Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T12:55:55.464363833Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:55:55.471297072Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-20T12:55:56.12049557Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:44412"
level=debug ts=2026-04-20T12:55:57.303090144Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=info ts=2026-04-20T12:55:57.751534029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:55:57.751560195Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:55:57.751570542Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:55:57.751568866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:01.333598299Z caller=checkpoint.go:378 msg="writing series" size="486 kB"
level=debug ts=2026-04-20T12:56:01.333624102Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=7
level=warn ts=2026-04-20T12:56:02.751610688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:05.242404376Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:56:05.362821937Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-20T12:56:07.751535352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:56:08.128148712Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-9jvfl-4eb65fd0 10.129.0.62:7946"
level=debug ts=2026-04-20T12:56:09.392615423Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-20T12:56:12.750956431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:13.423173728Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-20T12:56:17.452295268Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=warn ts=2026-04-20T12:56:17.751538821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:56:18.897626026Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:56:18.903187551Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:56:18.903205268Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:56:18.903227952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:56:18.907393757Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:56:21.482682891Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T12:56:22.751083135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:25.511928495Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=info ts=2026-04-20T12:56:26.208006232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:56:26.218517477Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-20T12:56:27.729001679Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:56:27.729009823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:56:27.751575667Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:56:27.751595269Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:56:27.751602944Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:56:27.75161946Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:29.542360292Z caller=checkpoint.go:378 msg="writing series" size="236 kB"
level=warn ts=2026-04-20T12:56:32.750894999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:33.571793157Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=info ts=2026-04-20T12:56:34.196797993Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:56:34.204298694Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=info ts=2026-04-20T12:56:34.86189021Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:56:34.867328778Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:56:37.60234416Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T12:56:37.750858807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:56:38.130019384Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:56:41.100404808Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:56:41.632373307Z caller=checkpoint.go:378 msg="writing series" size="757 kB"
level=debug ts=2026-04-20T12:56:41.632396647Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=10
level=warn ts=2026-04-20T12:56:42.751574176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:56:42.806699148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:56:42.811033108Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:56:43.969804297Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:56:45.661638079Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:56:47.75100362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:49.691649058Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=info ts=2026-04-20T12:56:51.085818273Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:56:51.093726224Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:56:52.751744159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:56:53.72080223Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-20T12:56:53.904386725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:56:53.908822051Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-20T12:56:56.122513082Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56544"
level=info ts=2026-04-20T12:56:57.750951242Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:56:57.750942515Z caller=checkpoint.go:390 msg="flushing series" totalSize="18 kB" series=3
level=debug ts=2026-04-20T12:56:57.750976589Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:56:57.750985662Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:56:57.750988032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:56:57.755211156Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-20T12:56:57.757257369Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006713606s
ts=2026-04-20T12:57:01.406459564Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:48340"
level=warn ts=2026-04-20T12:57:02.751617889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-20T12:57:07.751585282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:57:08.132658613Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
ts=2026-04-20T12:57:09.218339758Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:56354"
level=info ts=2026-04-20T12:57:12.209071739Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:57:12.21469574Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-20T12:57:12.751721724Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:13.428709701Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T12:57:17.751663495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:57:21.733580866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:57:21.740918073Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:57:22.751563989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:22.841496062Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:57:24.473471645Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:57:24.480826145Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:57:24.480846009Z caller=flush.go:240 component=ingester loop=0 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:57:24.480871413Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:57:24.487861622Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=info ts=2026-04-20T12:57:27.019802501Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:57:27.024043882Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:57:27.728785248Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:57:27.729857427Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:57:27.750264345Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-20T12:57:27.750376506Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-20T12:57:27.750558433Z caller=checkpoint.go:378 msg="writing series" size="172 kB"
level=info ts=2026-04-20T12:57:27.75135607Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T12:57:27.751362575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:27.751380768Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:57:27.751389174Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T12:57:31.787194033Z caller=checkpoint.go:378 msg="writing series" size="9.3 MB"
level=debug ts=2026-04-20T12:57:31.787218747Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.5 MB" series=2
level=warn ts=2026-04-20T12:57:32.751735227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:35.810707946Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=warn ts=2026-04-20T12:57:37.75127007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:57:38.135398581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:57:39.840197486Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-04-20T12:57:41.750068108Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:57:41.761318464Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T12:57:42.751165133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:43.266559968Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:57:43.870224858Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-20T12:57:46.750523249Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:57:46.756200353Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:57:47.750817206Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:47.900719602Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=info ts=2026-04-20T12:57:50.895125298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:57:50.903318638Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:57:50.903338796Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:57:50.903369087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:57:50.909923333Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:57:51.930203244Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-20T12:57:52.75095424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:55.959699096Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-20T12:57:56.126592066Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:44168"
level=info ts=2026-04-20T12:57:57.751444042Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:57:57.751473887Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:57:57.751483051Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:57:57.751531624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:57:59.990527651Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=info ts=2026-04-20T12:58:01.474418132Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:58:01.484563124Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-20T12:58:02.067328886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:58:02.074931186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-20T12:58:02.751625053Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:04.019895448Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=warn ts=2026-04-20T12:58:07.750826229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:58:07.995617511Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:58:08.003203157Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:58:08.0493816Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
level=debug ts=2026-04-20T12:58:08.049399201Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-04-20T12:58:08.137092593Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=info ts=2026-04-20T12:58:10.838261078Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:58:10.843060411Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:58:11.886303077Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T12:58:12.079903041Z caller=checkpoint.go:378 msg="writing series" size="814 B"
level=warn ts=2026-04-20T12:58:12.751202702Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:13.14831841Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-20T12:58:14.335373876Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:58:14.339529446Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:58:16.109179773Z caller=checkpoint.go:378 msg="writing series" size="550 kB"
level=warn ts=2026-04-20T12:58:17.75084711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:20.139276709Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-20T12:58:22.750882316Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:24.170673404Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-20T12:58:24.170695116Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=4
level=info ts=2026-04-20T12:58:26.195884144Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:58:26.205625955Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T12:58:27.728546244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:58:27.729607995Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-20T12:58:27.751136523Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:58:27.751154704Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:58:27.751163276Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:58:27.751152635Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:28.198736215Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=debug ts=2026-04-20T12:58:32.250805889Z caller=checkpoint.go:378 msg="writing series" size="23 MB"
level=debug ts=2026-04-20T12:58:32.250838051Z caller=checkpoint.go:390 msg="flushing series" totalSize="23 MB" series=2
level=warn ts=2026-04-20T12:58:32.751147318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:36.259003861Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-04-20T12:58:37.071538032Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:58:37.077855722Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:58:37.751246269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:58:38.138381688Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T12:58:38.52713091Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:58:40.288432634Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=warn ts=2026-04-20T12:58:42.751640808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:58:43.435198426Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:58:43.443263941Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:58:44.318164838Z caller=checkpoint.go:378 msg="writing series" size="541 kB"
level=warn ts=2026-04-20T12:58:47.751686743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:48.372211255Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-20T12:58:48.372237247Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=4
level=info ts=2026-04-20T12:58:48.90179922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=3 total_comp="9.0 MB" avg_comp="3.0 MB" total_uncomp="36 MB" avg_uncomp="12 MB" full=3 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:58:48.910806973Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:58:52.378357662Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=warn ts=2026-04-20T12:58:52.751737398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:58:56.407689849Z caller=checkpoint.go:378 msg="writing series" size="1.0 MB"
level=debug ts=2026-04-20T12:58:56.40771232Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=2
level=debug ts=2026-04-20T12:58:56.921654443Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-20T12:58:57.751637794Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:58:57.751666693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:58:57.751676946Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:58:57.751674891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:00.437583358Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=info ts=2026-04-20T12:59:00.652226906Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:59:00.658427861Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-20T12:59:01.417727802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.103:39256"
level=warn ts=2026-04-20T12:59:02.750832921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:59:03.461626008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:59:03.466349274Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T12:59:04.467834443Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-20T12:59:07.751100281Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:59:08.139977903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=debug ts=2026-04-20T12:59:08.497502085Z caller=checkpoint.go:378 msg="writing series" size="958 kB"
level=debug ts=2026-04-20T12:59:08.711710336Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T12:59:09.224407654Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:55638"
level=debug ts=2026-04-20T12:59:12.527302045Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T12:59:12.751273578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:16.557174399Z caller=checkpoint.go:378 msg="writing series" size="362 kB"
level=debug ts=2026-04-20T12:59:16.557198973Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=info ts=2026-04-20T12:59:17.192875203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:59:17.201974355Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T12:59:17.751382125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:20.586880238Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
level=warn ts=2026-04-20T12:59:22.751595315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:24.616906008Z caller=checkpoint.go:378 msg="writing series" size="148 kB"
level=info ts=2026-04-20T12:59:27.728922806Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T12:59:27.729374973Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T12:59:27.750868401Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:59:27.750887646Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:59:27.750897182Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:59:27.750909567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:28.646869193Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=info ts=2026-04-20T12:59:31.694045625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:59:31.702021434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:59:31.702047572Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:59:31.70208118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:59:31.708134419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:59:32.676673176Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-20T12:59:32.751159004Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:59:36.595311188Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:59:36.59940478Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:59:36.599423706Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T12:59:36.599447013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:59:36.602675294Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T12:59:36.706096652Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=warn ts=2026-04-20T12:59:37.750985116Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T12:59:38.142015322Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
ts=2026-04-20T12:59:39.226103769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.64:40168"
level=debug ts=2026-04-20T12:59:40.735993126Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-20T12:59:42.751224334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:59:43.693316353Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T12:59:43.700444595Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T12:59:44.214873806Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T12:59:44.766158779Z caller=checkpoint.go:378 msg="writing series" size="133 kB"
level=info ts=2026-04-20T12:59:45.695544957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T12:59:45.700100421Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-20T12:59:45.826722039Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T12:59:45.835849867Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsbl\"}"
level=debug ts=2026-04-20T12:59:45.954928026Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-20T12:59:45.956652024Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-20T12:59:47.751126621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:48.795980962Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=info ts=2026-04-20T12:59:51.552953506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T12:59:51.556900729Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-20T12:59:52.75088299Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T12:59:52.826252039Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=debug ts=2026-04-20T12:59:52.826273469Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=9
ts=2026-04-20T12:59:56.133605172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.62:56794"
level=info ts=2026-04-20T12:59:56.645223361Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T12:59:56.650002713Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T12:59:56.856115158Z caller=checkpoint.go:378 msg="writing series" size="441 kB"
level=info ts=2026-04-20T12:59:57.750868977Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T12:59:57.750894912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T12:59:57.750905881Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T12:59:57.750906501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T12:59:58.810420733Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=7 total_comp="20 MB" avg_comp="2.8 MB" total_uncomp="166 MB" avg_uncomp="24 MB" full=6 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T12:59:58.817471461Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 7, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T13:00:00.885049935Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-20T13:00:02.751364475Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:04.915312817Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
level=warn ts=2026-04-20T13:00:07.751343621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T13:00:08.14326176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T13:00:08.945471372Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-20T13:00:09.53835704Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-20T13:00:12.751600598Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:12.97540423Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=debug ts=2026-04-20T13:00:17.005042812Z caller=checkpoint.go:378 msg="writing series" size="337 kB"
level=debug ts=2026-04-20T13:00:17.005067609Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=6
level=warn ts=2026-04-20T13:00:17.75094176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T13:00:20.488900217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.7 MB" avg_comp="2.6 MB" total_uncomp="66 MB" avg_uncomp="22 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T13:00:20.499084663Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T13:00:21.034690114Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-20T13:00:22.751025156Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:25.06454889Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=info ts=2026-04-20T13:00:25.743944595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T13:00:25.748371527Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-20T13:00:27.728912785Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T13:00:27.728918511Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=debug ts=2026-04-20T13:00:27.730185505Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-20T13:00:27.730226852Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974099 n=1
level=debug ts=2026-04-20T13:00:27.730274528Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974099 n=1
level=info ts=2026-04-20T13:00:27.751607137Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-20T13:00:27.751625462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:27.751638433Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T13:00:27.751647532Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-20T13:00:29.094131781Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-04-20T13:00:30.218611375Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T13:00:30.226388409Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsbl\"}"
level=warn ts=2026-04-20T13:00:32.751314406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:33.126241764Z caller=checkpoint.go:378 msg="writing series" size="2.3 MB"
level=debug ts=2026-04-20T13:00:33.126269953Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=4
level=debug ts=2026-04-20T13:00:35.74465913Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T13:00:36.099953723Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T13:00:37.154272819Z caller=checkpoint.go:378 msg="writing series" size="640 B"
level=warn ts=2026-04-20T13:00:37.7512842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T13:00:38.14557287Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-qmrzd-23211a0d 10.128.0.103:7946"
level=info ts=2026-04-20T13:00:40.350519718Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T13:00:40.354841087Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T13:00:41.184103006Z caller=checkpoint.go:378 msg="writing series" size="501 kB"
level=warn ts=2026-04-20T13:00:42.751195314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T13:00:44.659968573Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T13:00:44.665852721Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T13:00:45.213885144Z caller=checkpoint.go:378 msg="writing series" size="467 kB"
level=info ts=2026-04-20T13:00:46.159749756Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.8 MB" total_uncomp="191 MB" avg_uncomp="24 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T13:00:46.168148504Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-20T13:00:47.751597139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:49.243887605Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-20T13:00:52.751366491Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:00:53.273410098Z caller=checkpoint.go:378 msg="writing series" size="39 kB"
level=debug ts=2026-04-20T13:00:57.303284176Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-04-20T13:00:57.303317349Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=info ts=2026-04-20T13:00:57.751224187Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T13:00:57.75126483Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T13:00:57.751273961Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T13:00:57.751260338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:01.333070205Z caller=checkpoint.go:378 msg="writing series" size="487 kB"
level=warn ts=2026-04-20T13:01:02.751041618Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:04.950801327Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-20T13:01:05.363434773Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
level=warn ts=2026-04-20T13:01:07.751720259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-20T13:01:08.149042957Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=debug ts=2026-04-20T13:01:09.393368078Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-20T13:01:12.750905981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:13.42287119Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=info ts=2026-04-20T13:01:15.698598784Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b453ab103cd7ea0 immediate=false num_chunks=4 total_comp="12 MB" avg_comp="3.0 MB" total_uncomp="47 MB" avg_uncomp="12 MB" full=4 labels="{stream=\"group-jsbl\"}"
level=error ts=2026-04-20T13:01:15.709088628Z caller=flush.go:262 component=ingester loop=0 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b453ab103cd7ea0/19daadca0d0%3A19daaeb5754%3A3e7ba3e3?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"group-jsbl\"}"
level=info ts=2026-04-20T13:01:16.544749886Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="298 kB" avg_comp="298 kB" total_uncomp="3.0 MB" avg_uncomp="3.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-20T13:01:16.549119315Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19daace67b6%3A19daacf95dc%3A6214bc7d?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-20T13:01:17.452969833Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-04-20T13:01:17.750949343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T13:01:19.649431213Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.6 MB" total_uncomp="88 MB" avg_uncomp="22 MB" full=4 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-20T13:01:19.656372268Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19daace6792%3A19daaceeb19%3A99660cab?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-20T13:01:21.482734977Z caller=checkpoint.go:378 msg="writing series" size="102 kB"
level=warn ts=2026-04-20T13:01:22.751638356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:25.512375232Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=debug ts=2026-04-20T13:01:25.512406916Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-20T13:01:27.728885638Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-20T13:01:27.729189262Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.69:11211 count=1
level=info ts=2026-04-20T13:01:27.750745317Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-20T13:01:27.750785882Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-20T13:01:27.750795101Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-20T13:01:27.750820965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-20T13:01:28.558305702Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T13:01:28.562900658Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T13:01:28.562928919Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-20T13:01:28.562959369Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T13:01:28.566749142Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-20T13:01:29.542596634Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=warn ts=2026-04-20T13:01:32.751120198Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:33.572074498Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-04-20T13:01:35.130374772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=8 total_comp="23 MB" avg_comp="2.8 MB" total_uncomp="191 MB" avg_uncomp="24 MB" full=7 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-20T13:01:35.137401592Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19daace6746%3A19daadf1ac1%3A2a2b81c9?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 8, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-20T13:01:36.111188146Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-20T13:01:37.602281048Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=warn ts=2026-04-20T13:01:37.751243325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:37.793604773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-20T13:01:38.150714625Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-9ff68b3e 10.129.0.64:7946"
level=info ts=2026-04-20T13:01:39.944035546Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.1 MB" avg_comp="1.1 MB" total_uncomp="5.9 MB" avg_uncomp="5.9 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-20T13:01:39.94885238Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19daace6ad0%3A19daad07484%3Ac348d838?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-20T13:01:41.63207316Z caller=checkpoint.go:378 msg="writing series" size="609 kB"
level=info ts=2026-04-20T13:01:41.865485764Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="542 kB" avg_comp="542 kB" total_uncomp="4.4 MB" avg_uncomp="4.4 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-20T13:01:41.869589877Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19daace6740%3A19daae8c779%3Aae039f87?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-20T13:01:42.750841285Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:45.661622597Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-20T13:01:47.751376604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:49.691037959Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-20T13:01:52.751510046Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-20T13:01:53.720992322Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"