level=info ts=2026-04-22T11:33:30.603935897Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-22T11:33:30.603957459Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-22T11:33:30.604338375Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-22T11:33:30.605050394Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-22T11:33:30.605429779Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T11:33:30.605521714Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.73 inf=eth0
level=info ts=2026-04-22T11:33:30.60601277Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-a2a7a25f
ts=2026-04-22T11:33:30.60608029Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-22T11:33:30.606093691Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=debug ts=2026-04-22T11:33:30.60611933Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.73 advertisePort=7946
level=info ts=2026-04-22T11:33:30.606268291Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
level=error ts=2026-04-22T11:33:30.609036195Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T11:33:30.609053632Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-22T11:33:30.609895088Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-22T11:33:30.610109599Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
ts=2026-04-22T11:33:30.610414266Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T11:33:30.610437505Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:33:30.610458346Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=4.191984ms
level=error ts=2026-04-22T11:33:30.610475435Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-22T11:33:30.610485454Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=info ts=2026-04-22T11:33:30.610620062Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T11:33:30.610645001Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T11:33:30.610642865Z
level=debug ts=2026-04-22T11:33:30.610661486Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-22T11:33:30.610678883Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-22T11:33:30.610694465Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-22T11:33:30.610699345Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-22T11:33:30.610698546Z
level=debug ts=2026-04-22T11:33:30.610703817Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=debug ts=2026-04-22T11:33:30.611660783Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-22T11:33:30.611746774Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.73 inf=eth0
ts=2026-04-22T11:33:30.61287158Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:33:30.612884251Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T11:33:30.625098854Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-22T11:33:30.625157974Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=info ts=2026-04-22T11:33:30.625170003Z caller=module_service.go:82 msg=starting module=runtime-config
level=debug ts=2026-04-22T11:33:30.625166841Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-22T11:33:30.625178524Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-22T11:33:30.625192272Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-22T11:33:30.62522621Z caller=module_service.go:82 msg=starting module=server
level=info ts=2026-04-22T11:33:30.625298781Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-22T11:33:30.625308596Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=debug ts=2026-04-22T11:33:30.625315059Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-22T11:33:30.625317574Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-22T11:33:30.62531993Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-22T11:33:30.625390535Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-22T11:33:30.625393615Z caller=module_service.go:82 msg=starting module=store
level=info ts=2026-04-22T11:33:30.625397399Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-22T11:33:30.625415025Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-22T11:33:30.625438561Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-22T11:33:30.625442355Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=debug ts=2026-04-22T11:33:30.625445749Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-22T11:33:30.625449903Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=debug ts=2026-04-22T11:33:30.625453263Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-22T11:33:30.625456129Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-22T11:33:30.625458917Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-22T11:33:30.625461661Z caller=module_service.go:82 msg=starting module=analytics
level=info ts=2026-04-22T11:33:30.625566769Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-22T11:33:30.625577854Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-22T11:33:30.625613023Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-22T11:33:30.625657871Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=80.537µs errors=false
level=info ts=2026-04-22T11:33:30.625674717Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-22T11:33:30.625830017Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=253.653µs errors=false
level=info ts=2026-04-22T11:33:30.62584018Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-22T11:33:30.625850562Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=274.32µs
level=info ts=2026-04-22T11:33:30.625881277Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-22T11:33:30.625894767Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=9.506669727s
level=info ts=2026-04-22T11:33:30.625904212Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-22T11:33:30.62591057Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-22T11:33:30.625911668Z caller=loki.go:599 msg="Loki started" startup_time=90.580625ms
level=info ts=2026-04-22T11:33:30.625921189Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=debug ts=2026-04-22T11:33:30.626005421Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-22T11:33:30.62601105Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-22T11:33:30.626095461Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-22T11:33:32.252460988Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:33:32.252493196Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:33:35.627096878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:33:36.077181304Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:33:36.077227028Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:33:40.626331967Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:33:41.024171099Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:33:41.024195711Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:33:45.626883059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:33:49.598517334Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:33:49.598541727Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=5 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:33:50.62628766Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:33:54.65231114Z caller=logging.go:137 msg="GET /ready (503) 119.698µs"
level=warn ts=2026-04-22T11:33:55.626018905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:34:00.62667972Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-22T11:34:00.626712503Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-22T11:34:00.626717887Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:34:00.626734714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:34:04.651343406Z caller=logging.go:137 msg="GET /ready (503) 96.152µs"
level=warn ts=2026-04-22T11:34:05.62691204Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:34:05.629035556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:34:06.030032669Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.71:7946: dial tcp 10.128.0.71:7946: connect: connection refused"
ts=2026-04-22T11:34:06.030952635Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.128.0.75:7946: dial tcp 10.128.0.75:7946: connect: connection refused"
ts=2026-04-22T11:34:06.031756156Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.74:7946"
ts=2026-04-22T11:34:06.033752438Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.81:7946"
ts=2026-04-22T11:34:06.035331316Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.129.0.83:7946"
ts=2026-04-22T11:34:06.036614202Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.80:7946: dial tcp 10.129.0.80:7946: connect: connection refused"
level=info ts=2026-04-22T11:34:06.036627166Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=35.426135107s
level=warn ts=2026-04-22T11:34:10.62638074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:34:15.626655483Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:34:20.625999179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:34:24.464552008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57866"
level=warn ts=2026-04-22T11:34:25.626099822Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:34:26.464138234Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T11:34:30.05370529Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:34:30.610270127Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:34:30.610545571Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T11:34:30.626806857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:34:30.626809534Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:34:30.626827975Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:34:30.626836246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:34:35.626705612Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:34:36.023318797Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T11:34:36.023338595Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:34:36.031103673Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T11:34:39.050674369Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51262"
level=warn ts=2026-04-22T11:34:40.626975924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:34:45.626995433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:34:45.685176829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=info ts=2026-04-22T11:34:50.27343377Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:34:50.280617351Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:34:50.626102054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:34:55.626327831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:35:00.055719634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=warn ts=2026-04-22T11:35:00.626739552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:35:00.626738438Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:35:00.62677132Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:35:00.62677907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:35:04.391031805Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T11:35:05.519148333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:35:05.626247704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:35:10.626751773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:35:15.62662971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:35:17.74726182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:35:17.754330445Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:35:20.625967071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:35:25.352393391Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:35:25.626153389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:35:30.05829216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:35:30.610253347Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:35:30.610266791Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T11:35:30.626631494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:35:30.626641179Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:35:30.626659871Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:35:30.626667715Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:35:32.60292524Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:35:35.626445327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:35:39.054871818Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58328"
level=warn ts=2026-04-22T11:35:40.626363633Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:35:45.626602817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:35:50.625989094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:35:52.495445171Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T11:35:52.497202496Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:35:55.626410104Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:36:00.060204495Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:36:00.626116953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:36:00.626147237Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T11:36:00.626154609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:36:00.626158868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:36:02.726829788Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:36:05.626170523Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:36:09.056742579Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56118"
level=warn ts=2026-04-22T11:36:10.62668419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:36:15.472409831Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:36:15.479487739Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:36:15.626747397Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:36:20.626072753Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:36:25.62598239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:36:28.859989822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-22T11:36:30.061967695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
ts=2026-04-22T11:36:30.557201407Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:39048"
level=info ts=2026-04-22T11:36:30.610272266Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:36:30.610643005Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:36:30.626920993Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T11:36:30.626928066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:36:30.626937736Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:36:30.62694585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:36:35.307771025Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:36:35.626453012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:36:39.058679207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36378"
level=warn ts=2026-04-22T11:36:40.626955187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:36:45.626692313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:36:50.625955039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:36:55.62686353Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:36:57.281534738Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:36:57.290181139Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T11:37:00.062808266Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:37:00.626187303Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:37:00.626227504Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:37:00.62623553Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:37:00.626226207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:37:04.976492666Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:37:05.626229123Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:37:09.060665281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50254"
level=warn ts=2026-04-22T11:37:10.626815261Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:37:15.626729302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:37:20.626099823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:37:22.273551855Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-22T11:37:25.627043344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:37:30.064486814Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
ts=2026-04-22T11:37:30.560591971Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51458"
level=info ts=2026-04-22T11:37:30.610291997Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:37:30.610621055Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:37:30.626897022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:37:30.626919362Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:37:30.626927308Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:37:30.626922864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:37:35.626823407Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:37:36.883148253Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:37:39.063354153Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43772"
level=warn ts=2026-04-22T11:37:40.626204161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:37:42.670101533Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:37:42.678498873Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:37:45.62611408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:37:50.626010567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:37:55.626812835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:38:00.066334348Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
ts=2026-04-22T11:38:00.562721566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34778"
level=info ts=2026-04-22T11:38:00.62608538Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:38:00.626109337Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:38:00.626117862Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:38:00.626118607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:38:05.626088624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:38:10.625972764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:38:12.414519017Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:38:15.625974094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:38:20.626968792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:38:25.625966003Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:38:30.067837088Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=debug ts=2026-04-22T11:38:30.610008237Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:38:30.611055453Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T11:38:30.626324668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:38:30.626352949Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:38:30.626361567Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T11:38:30.626357707Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T11:38:30.62641382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:38:30.62651776Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-22T11:38:30.626720234Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T11:38:34.456973241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:38:34.463255716Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:38:34.982359821Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=warn ts=2026-04-22T11:38:35.626862844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:38:36.781893309Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:47582"
level=debug ts=2026-04-22T11:38:37.531608386Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:38:37.945411591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T11:38:39.33717977Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T11:38:40.626812912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:38:43.691698259Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T11:38:45.626333387Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:38:48.046329935Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=warn ts=2026-04-22T11:38:50.626851881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:38:52.401533865Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T11:38:55.62647041Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:38:56.756398802Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
ts=2026-04-22T11:39:00.06961946Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=warn ts=2026-04-22T11:39:00.626147361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:39:00.626174663Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:39:00.626194971Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:39:00.626203486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:39:01.110730246Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=debug ts=2026-04-22T11:39:05.465464821Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T11:39:05.626816542Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:39:06.782972865Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42930"
level=debug ts=2026-04-22T11:39:07.360466944Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:39:09.069080457Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:52006"
level=debug ts=2026-04-22T11:39:09.820587013Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=warn ts=2026-04-22T11:39:10.626964334Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:14.175806894Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=warn ts=2026-04-22T11:39:15.627042197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:18.530737924Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-22T11:39:20.626112791Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:39:21.194946076Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:39:21.204057434Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:39:22.885564503Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:39:25.626159968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:27.240652116Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-22T11:39:30.07165192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
ts=2026-04-22T11:39:30.566986272Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49196"
level=debug ts=2026-04-22T11:39:30.610016107Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:39:30.611061885Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-22T11:39:30.626328718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:39:30.626331276Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:39:30.626348208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:39:30.62635665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:39:31.595119166Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-22T11:39:35.626622696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:35.949972045Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-04-22T11:39:40.304726206Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T11:39:40.626738481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:41.663526291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:39:44.659194277Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-04-22T11:39:45.626488186Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:49.014354355Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T11:39:50.626127719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:50.86597633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-22T11:39:53.369286508Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T11:39:55.62633113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:39:57.723855587Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-22T11:40:00.073780371Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
ts=2026-04-22T11:40:00.56857359Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40940"
level=info ts=2026-04-22T11:40:00.626259259Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:40:00.62629899Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T11:40:00.626294226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:40:00.626322979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:40:02.078834713Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-22T11:40:05.62692569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:06.433733138Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=debug ts=2026-04-22T11:40:06.433756029Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=23
level=info ts=2026-04-22T11:40:06.885172636Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:40:06.89565021Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:40:08.162162439Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:40:10.62618659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:10.788468267Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-22T11:40:15.142900779Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-22T11:40:15.626276697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:19.49854634Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T11:40:20.625976174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:23.853503116Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T11:40:25.627022141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:28.207542032Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
ts=2026-04-22T11:40:30.076223894Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
ts=2026-04-22T11:40:30.571076864Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41490"
level=info ts=2026-04-22T11:40:30.610491251Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:40:30.610887959Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:40:30.626093335Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:40:30.626109464Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:40:30.62611754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:40:30.626125466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:32.563150457Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-22T11:40:32.77180881Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:40:35.626339048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:40:36.788766283Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:38714"
level=debug ts=2026-04-22T11:40:36.917702733Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
ts=2026-04-22T11:40:39.073662789Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:55180"
level=warn ts=2026-04-22T11:40:40.626273899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:41.272377034Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-22T11:40:45.626068631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:45.62717607Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-22T11:40:49.982482373Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T11:40:50.626416147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:54.337374759Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=warn ts=2026-04-22T11:40:55.625946307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:40:58.692097236Z caller=checkpoint.go:378 msg="writing series" size="28 kB"
level=debug ts=2026-04-22T11:40:58.692123869Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
ts=2026-04-22T11:41:00.077656681Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:41:00.626657558Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:41:00.626685633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T11:41:00.626682255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:41:00.626705039Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:41:03.04711187Z caller=checkpoint.go:378 msg="writing series" size="466 kB"
level=debug ts=2026-04-22T11:41:04.221002829Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-22T11:41:04.22266306Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T11:41:04.800268717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:41:04.807467544Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:41:04.807485173Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T11:41:04.807506178Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:41:04.813113001Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:41:05.62664308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:07.402082811Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-22T11:41:07.402107007Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=2
level=debug ts=2026-04-22T11:41:07.851442105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:41:10.626810797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:11.756195109Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T11:41:15.625999749Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:16.111123197Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-22T11:41:17.048259873Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:41:17.057237265Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:41:20.466159795Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T11:41:20.62594768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:24.821179688Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-22T11:41:25.627014047Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:29.175492835Z caller=checkpoint.go:378 msg="writing series" size="452 kB"
ts=2026-04-22T11:41:30.079363161Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:41:30.610557548Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:41:30.61088054Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:41:30.626092106Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:41:30.626115395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:41:30.626123371Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:41:30.626124546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:33.53048816Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=debug ts=2026-04-22T11:41:35.559717773Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:41:35.62671889Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:37.885433653Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-22T11:41:40.62644477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:42.24003806Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-22T11:41:45.626510151Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:46.594742636Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T11:41:50.626476413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:50.950156168Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T11:41:51.6644521Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:41:51.674521963Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:41:55.305144499Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-04-22T11:41:55.62655792Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:41:58.072260373Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:41:59.663848208Z caller=checkpoint.go:378 msg="writing series" size="5.1 MB"
level=debug ts=2026-04-22T11:41:59.663873087Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.0 MB" series=12
ts=2026-04-22T11:42:00.081647872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:42:00.626765173Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:42:00.626789347Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:42:00.626799154Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:42:00.626812787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:01.068595006Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:42:04.014499035Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T11:42:05.626731441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:08.369174374Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-22T11:42:10.626135293Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:12.724126259Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T11:42:15.62698227Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:17.079489172Z caller=checkpoint.go:378 msg="writing series" size="272 kB"
level=warn ts=2026-04-22T11:42:20.62686145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:21.433619458Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=warn ts=2026-04-22T11:42:25.625960555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:25.788656969Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=debug ts=2026-04-22T11:42:27.632656497Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:42:30.083389741Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T11:42:30.143552635Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T11:42:30.610246184Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:42:30.611581385Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:42:30.626839573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:42:30.626859039Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:42:30.62686676Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:42:30.626873722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:34.498809017Z caller=checkpoint.go:378 msg="writing series" size="274 kB"
level=warn ts=2026-04-22T11:42:35.626281017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:42:36.796252528Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:60942"
level=debug ts=2026-04-22T11:42:38.853257662Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=warn ts=2026-04-22T11:42:40.626278741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:43.208393098Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T11:42:45.626904834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:47.562438962Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=info ts=2026-04-22T11:42:47.726064191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:42:47.734359598Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:42:50.626177624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:51.917385267Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=debug ts=2026-04-22T11:42:51.917409954Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=debug ts=2026-04-22T11:42:54.182907902Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T11:42:55.626857789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:42:56.272275145Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-22T11:42:57.086559694Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:43:00.08510754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:43:00.626299136Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:43:00.626320869Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:43:00.626329239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:43:00.626345559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:00.627396424Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 kB" series=1
level=info ts=2026-04-22T11:43:00.63271715Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-22T11:43:00.635810086Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009143587s
level=warn ts=2026-04-22T11:43:05.626547434Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:43:10.626444179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:43:15.626198981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:43:20.626954347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:43:25.626008088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:28.494565994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:43:30.086609354Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:43:30.137737917Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:43:30.146725328Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T11:43:30.610387565Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:43:30.610784674Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:43:30.626006372Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T11:43:30.626033369Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:43:30.626051972Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:43:30.626060485Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:43:30.626070153Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:43:30.626151858Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=debug ts=2026-04-22T11:43:30.626351501Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T11:43:34.982234974Z caller=checkpoint.go:378 msg="writing series" size="175 kB"
level=warn ts=2026-04-22T11:43:35.626590332Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:43:36.799683469Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45154"
ts=2026-04-22T11:43:39.085648937Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33092"
level=debug ts=2026-04-22T11:43:39.336686032Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T11:43:40.626463682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:43.691224534Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T11:43:45.626624441Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:46.192498222Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:43:48.046666582Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=warn ts=2026-04-22T11:43:50.626668774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:52.401372082Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T11:43:55.626904666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:43:56.75624693Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=debug ts=2026-04-22T11:43:59.303249645Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:44:00.089102467Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:44:00.584861097Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40810"
level=warn ts=2026-04-22T11:44:00.626303148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:44:00.626328996Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:44:00.626354877Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:44:00.626369487Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:44:01.110439824Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=debug ts=2026-04-22T11:44:05.46526528Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T11:44:05.626438356Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:44:06.801486478Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54362"
level=debug ts=2026-04-22T11:44:09.820354933Z caller=checkpoint.go:378 msg="writing series" size="263 kB"
level=warn ts=2026-04-22T11:44:10.626246757Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:14.175220636Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=warn ts=2026-04-22T11:44:15.626655288Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:44:16.381232354Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:44:16.388906881Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:44:18.530106226Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-22T11:44:20.626269562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:22.885248413Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:44:25.626312734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:27.24002695Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
ts=2026-04-22T11:44:30.091067723Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:44:30.586949219Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35122"
level=info ts=2026-04-22T11:44:30.610332761Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:44:30.610699824Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:44:30.625928918Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:44:30.625946283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T11:44:30.625946013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:44:30.625953913Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:44:31.594772594Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=debug ts=2026-04-22T11:44:34.149755934Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:44:35.626723583Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:35.949732023Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=debug ts=2026-04-22T11:44:35.949761848Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
ts=2026-04-22T11:44:36.802618169Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59390"
level=debug ts=2026-04-22T11:44:40.304653033Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T11:44:40.626475897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:44.120904357Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:44:44.659666445Z caller=checkpoint.go:378 msg="writing series" size="65 kB"
level=warn ts=2026-04-22T11:44:45.62682369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:49.013864384Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T11:44:50.626731516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:53.369258669Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T11:44:55.626759718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:44:57.723429907Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
ts=2026-04-22T11:45:00.093183948Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:45:00.626677212Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:45:00.626699845Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:45:00.626708379Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:45:00.626674913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:02.07815818Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-22T11:45:04.541836447Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:45:05.626749454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:45:06.231320958Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:45:06.23916134Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:45:06.433972014Z caller=checkpoint.go:378 msg="writing series" size="325 kB"
level=warn ts=2026-04-22T11:45:10.626760428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:10.787677676Z caller=checkpoint.go:378 msg="writing series" size="1.8 kB"
level=debug ts=2026-04-22T11:45:15.142684946Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-22T11:45:15.62644623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:19.497745734Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T11:45:20.626897395Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:23.85251907Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-22T11:45:24.323546413Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T11:45:24.325425601Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:45:25.626774672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:28.207318238Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
ts=2026-04-22T11:45:30.094301825Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=debug ts=2026-04-22T11:45:30.609795187Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:45:30.61086003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:45:30.612140146Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T11:45:30.612178939Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974286 n=1
level=debug ts=2026-04-22T11:45:30.61223375Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974286 n=1
level=info ts=2026-04-22T11:45:30.626109016Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:45:30.626126542Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:45:30.626136843Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:45:30.626133389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:32.562308654Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=warn ts=2026-04-22T11:45:35.626310396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:36.917820952Z caller=checkpoint.go:378 msg="writing series" size="336 kB"
level=debug ts=2026-04-22T11:45:36.917839988Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=debug ts=2026-04-22T11:45:36.994911069Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:45:39.092832952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51920"
level=warn ts=2026-04-22T11:45:40.626461201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:41.272006578Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-22T11:45:45.626340982Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:45.626384748Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=debug ts=2026-04-22T11:45:49.981622225Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T11:45:50.62679044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:45:54.336321579Z caller=checkpoint.go:378 msg="writing series" size="205 kB"
level=warn ts=2026-04-22T11:45:55.626749271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:45:57.734626671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.9 MB" avg_comp="2.9 MB" total_uncomp="25 MB" avg_uncomp="25 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:45:57.744257122Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:45:58.691515637Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
ts=2026-04-22T11:46:00.095420487Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
ts=2026-04-22T11:46:00.592450218Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:34586"
level=warn ts=2026-04-22T11:46:00.626780189Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:46:00.626780814Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:46:00.626801036Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:46:00.626808385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:46:03.046256433Z caller=checkpoint.go:378 msg="writing series" size="461 kB"
level=warn ts=2026-04-22T11:46:05.626387914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:07.402230957Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-22T11:46:07.4022557Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.1 MB" series=7
level=debug ts=2026-04-22T11:46:08.188840647Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:46:10.627028795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:11.755827866Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T11:46:15.626343279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:16.11115577Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-22T11:46:18.030252173Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:46:20.465798976Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T11:46:20.626284324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:24.820453231Z caller=checkpoint.go:378 msg="writing series" size="53 kB"
level=warn ts=2026-04-22T11:46:25.626321185Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:29.175740884Z caller=checkpoint.go:378 msg="writing series" size="476 kB"
ts=2026-04-22T11:46:30.097023075Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:46:30.610671597Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:46:30.61111479Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T11:46:30.626334993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:46:30.626338789Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:46:30.62635554Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:46:30.6263633Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:46:33.530457774Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=warn ts=2026-04-22T11:46:35.626538748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:37.885239341Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-22T11:46:38.051955333Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:46:40.626076617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:42.2402646Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-22T11:46:45.626968161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:46.59491028Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=info ts=2026-04-22T11:46:49.91339881Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:46:49.92193225Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:46:50.626266932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:50.949478314Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-22T11:46:55.304770829Z caller=checkpoint.go:378 msg="writing series" size="144 kB"
level=warn ts=2026-04-22T11:46:55.626263223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:46:59.666663389Z caller=checkpoint.go:378 msg="writing series" size="6.0 MB"
level=debug ts=2026-04-22T11:46:59.666813585Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.0 MB" series=12
ts=2026-04-22T11:47:00.099565097Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:47:00.59645974Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52750"
level=info ts=2026-04-22T11:47:00.625931099Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:47:00.625965178Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:47:00.625975314Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:47:00.625997516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:01.923201496Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:47:04.014290055Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T11:47:05.626432438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:47:06.810870649Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:37964"
level=debug ts=2026-04-22T11:47:08.368982513Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-22T11:47:09.124381105Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:47:10.625963797Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:12.723446284Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
level=warn ts=2026-04-22T11:47:15.626380049Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:17.078443751Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-22T11:47:20.626584722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:21.433127117Z caller=checkpoint.go:378 msg="writing series" size="86 kB"
level=warn ts=2026-04-22T11:47:25.626861255Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:25.788401599Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
ts=2026-04-22T11:47:30.101795579Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=debug ts=2026-04-22T11:47:30.142774363Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T11:47:30.61105971Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:47:30.611453575Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:47:30.626713166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T11:47:30.626719879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:30.626735127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:47:30.62674434Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:47:34.498274163Z caller=checkpoint.go:378 msg="writing series" size="298 kB"
level=info ts=2026-04-22T11:47:35.436528885Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:47:35.443870287Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:47:35.443889812Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T11:47:35.44391548Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:47:35.449669901Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:47:35.626667893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:36.743741636Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T11:47:36.81226408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51376"
level=debug ts=2026-04-22T11:47:38.852671318Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=warn ts=2026-04-22T11:47:40.626511708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:43.207734634Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=debug ts=2026-04-22T11:47:43.856513107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:47:45.626614986Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:47.562496137Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-22T11:47:50.626021536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:51.917513275Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=debug ts=2026-04-22T11:47:51.917540088Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=12
level=info ts=2026-04-22T11:47:53.239147971Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:47:53.246359375Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:47:55.62625161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:47:56.271899857Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T11:48:00.104178879Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T11:48:00.626368157Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 kB" series=1
level=warn ts=2026-04-22T11:48:00.626393679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:48:00.62640345Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:48:00.626420482Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:48:00.626432754Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T11:48:00.632195881Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-22T11:48:00.633538207Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00722442s
level=warn ts=2026-04-22T11:48:05.627065698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:48:06.81370697Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39052"
level=warn ts=2026-04-22T11:48:10.626400675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:11.821622101Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:48:15.626394345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:48:20.626784597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:48:25.626596463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:26.980678737Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T11:48:30.105612516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:48:30.396826417Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:48:30.403218973Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T11:48:30.600595578Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:52082"
level=info ts=2026-04-22T11:48:30.610243371Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:48:30.610484491Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:48:30.626748712Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T11:48:30.626783153Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T11:48:30.626783669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:30.626805391Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:48:30.626814333Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T11:48:30.626924807Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-22T11:48:30.627100281Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T11:48:34.982398221Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-22T11:48:35.626517896Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:48:36.814802507Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55314"
level=debug ts=2026-04-22T11:48:39.337435683Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T11:48:40.627019415Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:43.692265332Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=debug ts=2026-04-22T11:48:44.590783802Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:48:45.626265522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:48.047143093Z caller=checkpoint.go:378 msg="writing series" size="166 kB"
level=warn ts=2026-04-22T11:48:50.626410115Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:52.401548463Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T11:48:55.627073829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:48:56.756801593Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
ts=2026-04-22T11:49:00.107177774Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:49:00.626873393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:49:00.62690135Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:49:00.626910655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:49:00.626954783Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:01.11119636Z caller=checkpoint.go:378 msg="writing series" size="515 B"
level=debug ts=2026-04-22T11:49:05.466794857Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T11:49:05.626182481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:09.82173394Z caller=checkpoint.go:378 msg="writing series" size="355 kB"
level=warn ts=2026-04-22T11:49:10.625994026Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:14.175732657Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=debug ts=2026-04-22T11:49:14.200877331Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:49:15.626516199Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:18.531313187Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-04-22T11:49:20.626737824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:22.885529122Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:49:25.626630592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:27.240988152Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T11:49:28.551920683Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T11:49:28.553815673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T11:49:29.491360597Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:49:29.500014355Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T11:49:30.108973903Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
ts=2026-04-22T11:49:30.603632384Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:40290"
level=info ts=2026-04-22T11:49:30.610301526Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:49:30.610618121Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:49:30.626878578Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:49:30.626908465Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:49:30.626917481Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:49:30.626929131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:31.595107671Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
level=warn ts=2026-04-22T11:49:35.626224378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:35.950366138Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=debug ts=2026-04-22T11:49:35.950392756Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
ts=2026-04-22T11:49:36.817262394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57954"
level=debug ts=2026-04-22T11:49:40.304834952Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-22T11:49:40.626823668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:41.510959553Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:49:44.659897749Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-22T11:49:45.626013923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:49.014918405Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-22T11:49:50.626348179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:53.369293683Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-22T11:49:55.626760133Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:49:57.724445247Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
ts=2026-04-22T11:50:00.111381343Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:50:00.605952362Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46834"
level=info ts=2026-04-22T11:50:00.626753373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:50:00.626779737Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:50:00.626787375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:50:00.626778425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:02.079393962Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
level=warn ts=2026-04-22T11:50:05.626709082Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:06.434291985Z caller=checkpoint.go:378 msg="writing series" size="225 kB"
level=debug ts=2026-04-22T11:50:09.178183499Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:50:10.626910137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:10.789199194Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=debug ts=2026-04-22T11:50:15.144309023Z caller=checkpoint.go:378 msg="writing series" size="254 kB"
level=warn ts=2026-04-22T11:50:15.626004143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:17.023572135Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:50:19.498992116Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T11:50:20.626804645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:23.853496635Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T11:50:25.626711307Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:50:27.935529099Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:50:27.946092706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:50:28.208020831Z caller=checkpoint.go:378 msg="writing series" size="259 kB"
level=debug ts=2026-04-22T11:50:28.208045935Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
ts=2026-04-22T11:50:30.113886462Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:50:30.607997904Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:36620"
level=info ts=2026-04-22T11:50:30.610425773Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:50:30.610781307Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:50:30.626029986Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:50:30.626043888Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T11:50:30.626043898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:50:30.626053006Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:50:32.562647409Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=warn ts=2026-04-22T11:50:35.626775893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:36.918010376Z caller=checkpoint.go:378 msg="writing series" size="463 kB"
level=debug ts=2026-04-22T11:50:37.176513374Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:50:40.626673813Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:41.272582386Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-04-22T11:50:45.627057431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:45.627125427Z caller=checkpoint.go:378 msg="writing series" size="5.1 kB"
level=debug ts=2026-04-22T11:50:49.983048561Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T11:50:50.626701608Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:54.337292911Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-22T11:50:54.33732053Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T11:50:55.626163376Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:50:58.692458778Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
ts=2026-04-22T11:51:00.116371769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
ts=2026-04-22T11:51:00.609971798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:41066"
level=info ts=2026-04-22T11:51:00.626695381Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:51:00.626717867Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:51:00.626726179Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:51:00.62672951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:03.047312824Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=warn ts=2026-04-22T11:51:05.626343252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:07.40235272Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-22T11:51:07.402381908Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-04-22T11:51:10.626563595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:11.756841088Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=debug ts=2026-04-22T11:51:12.246310579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:51:12.594195027Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T11:51:15.12712404Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:51:15.133292191Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:51:15.626448438Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:16.111584735Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-22T11:51:20.466832822Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T11:51:20.626271038Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:24.821262481Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-22T11:51:25.626594221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:29.177362671Z caller=checkpoint.go:378 msg="writing series" size="2.7 MB"
level=debug ts=2026-04-22T11:51:29.177386936Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.9 MB" series=5
ts=2026-04-22T11:51:30.11839897Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:51:30.610976586Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:51:30.611320596Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:51:30.626519632Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:51:30.626535455Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:51:30.626544217Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:51:30.626570201Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:33.531289633Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=warn ts=2026-04-22T11:51:35.6262548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:51:36.823386493Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:59490"
level=debug ts=2026-04-22T11:51:37.306199384Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:51:37.885569818Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
ts=2026-04-22T11:51:39.117004498Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36826"
level=warn ts=2026-04-22T11:51:40.626850107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:42.240526198Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
level=warn ts=2026-04-22T11:51:45.62600137Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:46.595277692Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-22T11:51:50.626130369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:50.950226806Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-22T11:51:55.305518528Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T11:51:55.334618954Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:51:55.342280199Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:51:55.626713007Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:51:59.663450357Z caller=checkpoint.go:378 msg="writing series" size="6.7 MB"
level=debug ts=2026-04-22T11:51:59.663474023Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.2 MB" series=7
ts=2026-04-22T11:52:00.120424786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
ts=2026-04-22T11:52:00.612979998Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46402"
level=info ts=2026-04-22T11:52:00.626339177Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:52:00.626358104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:52:00.626365518Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:52:00.626395068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:04.014500347Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-22T11:52:05.626089429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:52:06.824372331Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:55906"
level=debug ts=2026-04-22T11:52:08.370031514Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-22T11:52:09.872639724Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T11:52:10.626464912Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:10.998423132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:52:12.724546847Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=warn ts=2026-04-22T11:52:15.626689719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:17.079571636Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
level=warn ts=2026-04-22T11:52:20.626252121Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:21.434039577Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=warn ts=2026-04-22T11:52:25.626656575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:25.788517409Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-04-22T11:52:30.122281289Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T11:52:30.144244645Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T11:52:30.610678644Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:52:30.611093794Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
ts=2026-04-22T11:52:30.614924018Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53362"
level=info ts=2026-04-22T11:52:30.626306282Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:52:30.626321053Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:52:30.626328325Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:52:30.626353697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:34.498653253Z caller=checkpoint.go:378 msg="writing series" size="339 kB"
level=debug ts=2026-04-22T11:52:34.498675289Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=debug ts=2026-04-22T11:52:35.562246441Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:52:35.626276676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:38.853790631Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
ts=2026-04-22T11:52:39.121480093Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35284"
level=warn ts=2026-04-22T11:52:40.62624653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:43.208429798Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T11:52:45.626078428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:47.562765037Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
level=debug ts=2026-04-22T11:52:48.809581871Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T11:52:50.626269265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:51.918458184Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
level=info ts=2026-04-22T11:52:55.039726333Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:52:55.04708533Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:52:55.626668924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:52:56.272765747Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T11:53:00.124273832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
ts=2026-04-22T11:53:00.616736715Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58318"
level=info ts=2026-04-22T11:53:00.626228311Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:53:00.626246192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:53:00.626254261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:53:00.626272014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:00.627360036Z caller=checkpoint.go:390 msg="flushing series" totalSize="580 kB" series=5
level=info ts=2026-04-22T11:53:00.633285971Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-22T11:53:00.634685906Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007622508s
level=debug ts=2026-04-22T11:53:05.139320944Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T11:53:05.139387037Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:53:05.146343902Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:53:05.626243611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:53:10.626132422Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:53:15.626442963Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:53:16.249176285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:53:16.258720589Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:53:20.626333226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:53:25.626007576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:53:30.126116689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:53:30.611062462Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:53:30.611502566Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
ts=2026-04-22T11:53:30.618466706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:55580"
level=info ts=2026-04-22T11:53:30.626865896Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T11:53:30.626891879Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:53:30.626897128Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:53:30.626914229Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:53:30.626922806Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T11:53:30.627008959Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=debug ts=2026-04-22T11:53:30.627247008Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T11:53:31.844669039Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:53:34.781450746Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-22T11:53:35.626920471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:38.935683339Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=info ts=2026-04-22T11:53:39.071124769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:53:39.077671476Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T11:53:39.126572855Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41552"
level=warn ts=2026-04-22T11:53:40.62683279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:42.226963815Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T11:53:42.22884719Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T11:53:43.089659568Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T11:53:45.626704155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:47.243537362Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=warn ts=2026-04-22T11:53:50.62663242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:51.396901788Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T11:53:52.905433214Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:53:52.916643227Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:53:55.550810693Z caller=checkpoint.go:378 msg="writing series" size="118 kB"
level=warn ts=2026-04-22T11:53:55.626704064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:53:59.704673995Z caller=checkpoint.go:378 msg="writing series" size="515 B"
ts=2026-04-22T11:54:00.127925647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=warn ts=2026-04-22T11:54:00.626113851Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:54:00.626121571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:54:00.626143889Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:54:00.626157874Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:54:03.858243281Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=debug ts=2026-04-22T11:54:04.288415654Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:54:05.626725229Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:54:06.831744849Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52470"
level=debug ts=2026-04-22T11:54:08.013005743Z caller=checkpoint.go:378 msg="writing series" size="240 kB"
ts=2026-04-22T11:54:09.128673226Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42016"
level=warn ts=2026-04-22T11:54:10.626818936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:12.166343805Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-22T11:54:15.627105413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:16.32047152Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T11:54:16.655611519Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:54:20.473397679Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:54:20.626771494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:24.627925741Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T11:54:25.626982704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:28.781727216Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
ts=2026-04-22T11:54:30.129114695Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:54:30.6104819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:54:30.611177421Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:54:30.626536371Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:54:30.62657247Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:54:30.626581664Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:54:30.626664369Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:32.393809591Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:54:32.935470552Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=debug ts=2026-04-22T11:54:32.935497431Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-04-22T11:54:35.626967803Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:54:36.243958208Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:54:36.25348126Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T11:54:37.089643692Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-22T11:54:39.131178872Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:48164"
level=warn ts=2026-04-22T11:54:40.626820141Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:41.24335994Z caller=checkpoint.go:378 msg="writing series" size="85 kB"
level=info ts=2026-04-22T11:54:42.179738734Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:54:42.186610465Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:54:42.186634392Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T11:54:42.186664391Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:54:42.193584976Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:54:45.396717081Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-22T11:54:45.626098875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:49.550799174Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-22T11:54:50.626297764Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:54:53.70445274Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:54:55.626850083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:54:55.899092252Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:54:55.908874727Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:54:57.858416832Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
ts=2026-04-22T11:55:00.129946585Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:55:00.626347139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:55:00.626377098Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:55:00.626387176Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:55:00.626391675Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:01.697007612Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:55:02.01211257Z caller=checkpoint.go:378 msg="writing series" size="26 kB"
level=debug ts=2026-04-22T11:55:04.523301581Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:55:05.626220847Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:06.166841309Z caller=checkpoint.go:378 msg="writing series" size="361 kB"
level=debug ts=2026-04-22T11:55:10.319599333Z caller=checkpoint.go:378 msg="writing series" size="2.3 kB"
level=warn ts=2026-04-22T11:55:10.626735328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:14.473480564Z caller=checkpoint.go:378 msg="writing series" size="136 kB"
level=warn ts=2026-04-22T11:55:15.626565497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:18.628169781Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-04-22T11:55:20.597178656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:55:20.604664798Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:55:20.626497439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:55:20.806503637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:55:20.812882026Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:55:22.781375223Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T11:55:25.626704218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:26.9358899Z caller=checkpoint.go:378 msg="writing series" size="63 kB"
ts=2026-04-22T11:55:30.131477747Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:55:30.610234554Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:55:30.610245457Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:55:30.626640386Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:55:30.626656087Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:55:30.626664256Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:55:30.626665142Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:31.089250991Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-22T11:55:31.089272665Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-04-22T11:55:35.243347955Z caller=checkpoint.go:378 msg="writing series" size="606 kB"
level=debug ts=2026-04-22T11:55:35.251195732Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:55:35.626197882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:55:36.83725542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:52630"
ts=2026-04-22T11:55:39.134575246Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:41778"
level=debug ts=2026-04-22T11:55:39.397547813Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=warn ts=2026-04-22T11:55:40.626517333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:41.76415782Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:55:43.551249715Z caller=checkpoint.go:378 msg="writing series" size="6.4 kB"
level=warn ts=2026-04-22T11:55:45.626384849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:47.704822979Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T11:55:50.626863209Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:51.858751095Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-04-22T11:55:51.858779715Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-22T11:55:55.626186645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:55:56.01262764Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
ts=2026-04-22T11:56:00.13335146Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=debug ts=2026-04-22T11:56:00.166548307Z caller=checkpoint.go:378 msg="writing series" size="360 kB"
level=debug ts=2026-04-22T11:56:00.172535847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:56:00.626843057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:56:00.626863665Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:56:00.626882865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:56:00.626890912Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T11:56:04.321129404Z caller=checkpoint.go:378 msg="writing series" size="1.2 MB"
level=debug ts=2026-04-22T11:56:04.321153218Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=3
level=warn ts=2026-04-22T11:56:05.626521439Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:08.473957704Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T11:56:10.626259069Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:12.627411083Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-22T11:56:14.337102625Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:56:14.346057242Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T11:56:15.626127794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:16.781636293Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=info ts=2026-04-22T11:56:18.112422422Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:56:18.1185609Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:56:20.626146456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:20.935475053Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=debug ts=2026-04-22T11:56:25.092473132Z caller=checkpoint.go:378 msg="writing series" size="4.3 MB"
level=debug ts=2026-04-22T11:56:25.092497141Z caller=checkpoint.go:390 msg="flushing series" totalSize="4.5 MB" series=5
level=warn ts=2026-04-22T11:56:25.626442367Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:29.242980269Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
ts=2026-04-22T11:56:30.135414742Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:56:30.610673774Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:56:30.611113778Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:56:30.626442432Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:56:30.626459992Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:56:30.626468021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:56:30.626496447Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:32.004830787Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:56:33.39651682Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-04-22T11:56:35.626690474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:56:36.840149744Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:39458"
level=debug ts=2026-04-22T11:56:37.55045154Z caller=checkpoint.go:378 msg="writing series" size="6.5 kB"
ts=2026-04-22T11:56:39.138413091Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:60530"
level=warn ts=2026-04-22T11:56:40.626944866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:41.704592679Z caller=checkpoint.go:378 msg="writing series" size="73 kB"
level=debug ts=2026-04-22T11:56:43.387393591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T11:56:45.626696499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:45.858912145Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=debug ts=2026-04-22T11:56:50.011885882Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-22T11:56:50.626161173Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:54.166262042Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-22T11:56:55.626721012Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:56:58.323059185Z caller=checkpoint.go:378 msg="writing series" size="7.4 MB"
level=debug ts=2026-04-22T11:56:58.323082252Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.0 MB" series=8
level=debug ts=2026-04-22T11:56:59.133008519Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T11:57:00.137081503Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:57:00.626676855Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:57:00.626704693Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:57:00.626713003Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:57:00.62672812Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:02.474224099Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-04-22T11:57:02.635881668Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:57:02.644415353Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-22T11:57:04.14026223Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:57:04.146347052Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:57:05.626182175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:06.627744185Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-22T11:57:09.140123656Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:40182"
level=warn ts=2026-04-22T11:57:10.62600864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:10.781513616Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=debug ts=2026-04-22T11:57:14.935769583Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=warn ts=2026-04-22T11:57:15.626263992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:19.089178462Z caller=checkpoint.go:378 msg="writing series" size="191 kB"
level=warn ts=2026-04-22T11:57:20.626632236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:23.243183799Z caller=checkpoint.go:378 msg="writing series" size="227 kB"
level=warn ts=2026-04-22T11:57:25.626981754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:27.396746749Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
ts=2026-04-22T11:57:30.138734691Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:57:30.610238407Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:57:30.610525963Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:57:30.626929679Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:57:30.62694778Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:57:30.626955809Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:57:30.626953075Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:31.550363184Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T11:57:32.618644774Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:57:32.664519926Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T11:57:32.666002668Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T11:57:35.626133695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:35.704605876Z caller=checkpoint.go:378 msg="writing series" size="433 kB"
level=debug ts=2026-04-22T11:57:35.704629073Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
ts=2026-04-22T11:57:36.843309388Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49448"
level=debug ts=2026-04-22T11:57:39.858060099Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-22T11:57:40.626979324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:44.012360524Z caller=checkpoint.go:378 msg="writing series" size="58 kB"
level=warn ts=2026-04-22T11:57:45.625974716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:48.168502793Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T11:57:50.626081059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:52.319619158Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=warn ts=2026-04-22T11:57:55.626009707Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:57:56.474043058Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=debug ts=2026-04-22T11:57:58.262109423Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T11:57:59.257523891Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:57:59.269174385Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T11:57:59.665401398Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:57:59.672011709Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T11:58:00.140825689Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:58:00.625937024Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:58:00.62596486Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:58:00.625973701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:58:00.62596143Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:00.628048455Z caller=checkpoint.go:390 msg="flushing series" totalSize="527 kB" series=5
level=info ts=2026-04-22T11:58:00.631543628Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
ts=2026-04-22T11:58:00.63213901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38290"
level=info ts=2026-04-22T11:58:00.634462819Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007288866s
level=warn ts=2026-04-22T11:58:05.62631248Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:58:09.143463296Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:54192"
level=warn ts=2026-04-22T11:58:10.626698507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:58:15.626947242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T11:58:20.626636497Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:22.353988802Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:58:25.5284812Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T11:58:25.626237145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:58:30.143304994Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T11:58:30.610264005Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:58:30.610848044Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:58:30.626220795Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T11:58:30.626242772Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:58:30.626261073Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:58:30.626269436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:58:30.626319263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:58:30.62636824Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-22T11:58:30.626572704Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T11:58:30.634319694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48134"
level=debug ts=2026-04-22T11:58:34.78139218Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-04-22T11:58:35.626245377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:38.934350698Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-22T11:58:39.145743203Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51744"
level=warn ts=2026-04-22T11:58:40.626928343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:43.089144659Z caller=checkpoint.go:378 msg="writing series" size="6.0 kB"
level=warn ts=2026-04-22T11:58:45.626397867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:47.242401565Z caller=checkpoint.go:378 msg="writing series" size="220 kB"
level=info ts=2026-04-22T11:58:49.516440392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="22 MB" avg_uncomp="22 MB" synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:58:49.523529352Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:58:50.626558754Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:51.39667486Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=info ts=2026-04-22T11:58:52.072948109Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:58:52.079557736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:58:54.439533598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:58:55.55022591Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-22T11:58:55.626816824Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:58:59.704343378Z caller=checkpoint.go:378 msg="writing series" size="515 B"
ts=2026-04-22T11:59:00.145231598Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T11:59:00.626361571Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:59:00.626387283Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:59:00.626395386Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:59:00.626414114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:59:00.636640481Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:45770"
level=debug ts=2026-04-22T11:59:03.857983831Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T11:59:05.626088547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:59:06.848233007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:42172"
level=debug ts=2026-04-22T11:59:08.011985629Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
ts=2026-04-22T11:59:09.147524222Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:35550"
level=warn ts=2026-04-22T11:59:10.625963607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:12.165418632Z caller=checkpoint.go:378 msg="writing series" size="280 kB"
level=debug ts=2026-04-22T11:59:12.165446449Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-04-22T11:59:15.62692918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:16.319165141Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-22T11:59:17.713613822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T11:59:20.473094609Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T11:59:20.626529182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:20.627663225Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:59:24.627134104Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T11:59:25.627022125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:28.78087346Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
ts=2026-04-22T11:59:30.147155563Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T11:59:30.610816533Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T11:59:30.611760486Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T11:59:30.626076675Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T11:59:30.626101306Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T11:59:30.626109209Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T11:59:30.626117203Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T11:59:30.638495244Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:48298"
level=debug ts=2026-04-22T11:59:32.934765921Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=info ts=2026-04-22T11:59:35.52562698Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:59:35.538081767Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:59:35.538108497Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T11:59:35.538136008Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:59:35.545605627Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:59:35.626746148Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:37.088407551Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
ts=2026-04-22T11:59:39.149404371Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33208"
level=warn ts=2026-04-22T11:59:40.62698918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:41.24204694Z caller=checkpoint.go:378 msg="writing series" size="98 kB"
level=debug ts=2026-04-22T11:59:45.396567414Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-22T11:59:45.626865514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:49.550788766Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-22T11:59:50.626278898Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T11:59:51.006762129Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T11:59:51.017088497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T11:59:51.5384736Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T11:59:53.703813294Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=info ts=2026-04-22T11:59:55.186493096Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T11:59:55.192534614Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T11:59:55.626937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T11:59:57.857821595Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
ts=2026-04-22T12:00:00.149337064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:00:00.626266649Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:00:00.626295072Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:00:00.62630355Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:00:00.626328897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:00:00.64094788Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:47376"
level=debug ts=2026-04-22T12:00:02.011503232Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-22T12:00:05.626159219Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:06.165351062Z caller=checkpoint.go:378 msg="writing series" size="265 kB"
level=debug ts=2026-04-22T12:00:06.536316481Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T12:00:10.318946855Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=warn ts=2026-04-22T12:00:10.626006481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:14.47326132Z caller=checkpoint.go:378 msg="writing series" size="368 kB"
level=debug ts=2026-04-22T12:00:14.47328791Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=15
level=warn ts=2026-04-22T12:00:15.626507217Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:18.627175834Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T12:00:20.626152247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:22.780764207Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T12:00:25.626593331Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:26.782831849Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:00:26.934495969Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
ts=2026-04-22T12:00:30.151333244Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:00:30.610457412Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:00:30.61087924Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=debug ts=2026-04-22T12:00:30.612084644Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T12:00:30.612123723Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974287 n=1
level=debug ts=2026-04-22T12:00:30.612164113Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974287 n=1
level=info ts=2026-04-22T12:00:30.626259573Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T12:00:30.626265796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:30.626282625Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:00:30.626290413Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:00:31.089126798Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-04-22T12:00:32.556698116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:00:32.565311829Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:00:35.243298934Z caller=checkpoint.go:378 msg="writing series" size="572 kB"
level=warn ts=2026-04-22T12:00:35.626378715Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:39.396635764Z caller=checkpoint.go:378 msg="writing series" size="313 kB"
level=debug ts=2026-04-22T12:00:39.396668561Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=6
level=warn ts=2026-04-22T12:00:40.62597089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:00:41.0464498Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:00:41.052982385Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:00:43.54989387Z caller=checkpoint.go:378 msg="writing series" size="7.3 kB"
level=warn ts=2026-04-22T12:00:45.626321575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:47.703546238Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T12:00:50.626078345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:51.858187498Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=debug ts=2026-04-22T12:00:53.495579783Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:00:55.626737601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:00:56.011268906Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=debug ts=2026-04-22T12:00:59.633467439Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T12:01:00.153395556Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T12:01:00.165427615Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
level=info ts=2026-04-22T12:01:00.626755778Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:01:00.626784104Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:01:00.626792337Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:01:00.626786532Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:01:00.64365856Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38366"
level=debug ts=2026-04-22T12:01:04.320469503Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-22T12:01:04.320493824Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.2 MB" series=6
level=warn ts=2026-04-22T12:01:05.626955467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:01:06.85627212Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53162"
level=debug ts=2026-04-22T12:01:08.473715142Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T12:01:10.62607701Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:12.626619487Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-22T12:01:15.626546157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:16.781077459Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T12:01:20.626048577Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:20.935268299Z caller=checkpoint.go:378 msg="writing series" size="456 kB"
level=info ts=2026-04-22T12:01:22.204972497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:01:22.213923436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:01:22.213944614Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T12:01:22.21397233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:01:22.220192164Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:01:22.439845036Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:01:25.092809119Z caller=checkpoint.go:378 msg="writing series" size="6.4 MB"
level=debug ts=2026-04-22T12:01:25.092833631Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.0 MB" series=5
level=warn ts=2026-04-22T12:01:25.626708684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:29.24281025Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
ts=2026-04-22T12:01:30.155417671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:01:30.19511634Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:01:30.203178232Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T12:01:30.610304687Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:01:30.610952775Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:01:30.626298837Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:01:30.626337719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:01:30.626352087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:01:30.62634384Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:33.396813474Z caller=checkpoint.go:378 msg="writing series" size="163 kB"
level=warn ts=2026-04-22T12:01:35.627017396Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:37.549978055Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
ts=2026-04-22T12:01:39.15587534Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:58662"
level=info ts=2026-04-22T12:01:39.878187556Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:01:39.885200882Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:01:40.626458526Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:41.704159349Z caller=checkpoint.go:378 msg="writing series" size="81 kB"
level=warn ts=2026-04-22T12:01:45.626825802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:45.858065247Z caller=checkpoint.go:378 msg="writing series" size="121 kB"
level=debug ts=2026-04-22T12:01:48.569013185Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T12:01:48.570762425Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T12:01:50.012046174Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-22T12:01:50.626065872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:54.165431153Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=debug ts=2026-04-22T12:01:55.416779454Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:01:55.626619985Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:01:58.32377004Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-22T12:01:58.323796554Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.2 MB" series=8
ts=2026-04-22T12:02:00.157717926Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=warn ts=2026-04-22T12:02:00.626314656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:02:00.626312661Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:02:00.626341602Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:02:00.626350937Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:02:01.848527832Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:02:01.858079755Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:02:02.473515033Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-22T12:02:05.626157048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:06.627158437Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-22T12:02:10.626904456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:10.780957674Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-04-22T12:02:14.935354118Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-22T12:02:15.625968673Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:19.088749483Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=info ts=2026-04-22T12:02:19.877945199Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:02:19.884748591Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:02:20.62668902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:23.242834386Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-22T12:02:25.626165196Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:27.3960893Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-22T12:02:29.880806106Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T12:02:30.16032064Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:02:30.610286467Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:02:30.610456936Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T12:02:30.626934388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:02:30.626959965Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:02:30.62697519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:02:30.626982479Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:02:30.985047886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:02:31.550436235Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-22T12:02:35.626442597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:35.704272534Z caller=checkpoint.go:378 msg="writing series" size="506 kB"
level=debug ts=2026-04-22T12:02:35.704288679Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-22T12:02:39.857573818Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-22T12:02:40.626481601Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:44.012070365Z caller=checkpoint.go:378 msg="writing series" size="64 kB"
level=warn ts=2026-04-22T12:02:45.626693843Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:48.16605857Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-22T12:02:50.626686841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:52.31908233Z caller=checkpoint.go:378 msg="writing series" size="397 kB"
level=info ts=2026-04-22T12:02:52.410389514Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:02:52.418431044Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:02:55.626829995Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:02:56.473275557Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T12:03:00.162548777Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:03:00.57296002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:03:00.579662349Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:03:00.62677115Z caller=checkpoint.go:390 msg="flushing series" totalSize="726 kB" series=5
level=warn ts=2026-04-22T12:03:00.626769913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:03:00.626794285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:03:00.626812633Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:03:00.626822495Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:03:00.724655389Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-22T12:03:00.726385745Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.099851181s
level=warn ts=2026-04-22T12:03:05.626244031Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:03:05.74516629Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T12:03:06.864507007Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:54944"
ts=2026-04-22T12:03:09.161064689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:50190"
level=warn ts=2026-04-22T12:03:10.626719362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:03:13.081741102Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:03:15.626597647Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T12:03:20.626684952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T12:03:25.626265383Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:03:30.165194532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:03:30.610587532Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:03:30.611020799Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T12:03:30.626378009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:03:30.626386223Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-22T12:03:30.626402672Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-22T12:03:30.62640709Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:03:30.626417237Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:03:30.626515409Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-22T12:03:30.626662983Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T12:03:34.781523317Z caller=checkpoint.go:378 msg="writing series" size="203 kB"
level=warn ts=2026-04-22T12:03:35.625971545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:03:36.865676442Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46558"
level=debug ts=2026-04-22T12:03:38.934798089Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-22T12:03:39.16264601Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45216"
level=debug ts=2026-04-22T12:03:40.511716828Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:03:40.626974222Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:03:41.220385314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:03:41.23015449Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:03:43.088985052Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-22T12:03:45.626476735Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:03:47.243157586Z caller=checkpoint.go:378 msg="writing series" size="549 kB"
level=info ts=2026-04-22T12:03:49.403801949Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:03:49.409806579Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:03:50.626809535Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:03:51.396943099Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T12:03:55.550356485Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=warn ts=2026-04-22T12:03:55.626456092Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:03:59.70424689Z caller=checkpoint.go:378 msg="writing series" size="515 B"
ts=2026-04-22T12:04:00.166987107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:04:00.626918626Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T12:04:00.626918676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:00.62695674Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:04:00.626965225Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:04:03.857970813Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T12:04:05.62650607Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:06.848025139Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:04:08.012133292Z caller=checkpoint.go:378 msg="writing series" size="445 kB"
level=debug ts=2026-04-22T12:04:08.01215957Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=10
level=debug ts=2026-04-22T12:04:08.903538643Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T12:04:09.164312835Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:49122"
level=warn ts=2026-04-22T12:04:10.626707529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:12.166250602Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-22T12:04:15.626272033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:16.319272586Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=debug ts=2026-04-22T12:04:20.473427692Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T12:04:20.626322538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:04:23.827465288Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:04:23.835519102Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:04:24.627662356Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T12:04:25.6263368Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:28.781451557Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
ts=2026-04-22T12:04:30.169235488Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:04:30.61038539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:04:30.610579807Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:04:30.626001478Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T12:04:30.626009529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:30.626025855Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:04:30.626043585Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-22T12:04:30.656261798Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:35734"
level=debug ts=2026-04-22T12:04:32.935329367Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=warn ts=2026-04-22T12:04:35.626291787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:36.064754406Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:04:37.088752668Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T12:04:37.49697103Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=2 total_comp="5.4 MB" avg_comp="2.7 MB" total_uncomp="46 MB" avg_uncomp="23 MB" full=1 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:04:37.505260423Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:04:40.626244265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:41.242974065Z caller=checkpoint.go:378 msg="writing series" size="111 kB"
level=debug ts=2026-04-22T12:04:45.396784168Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-22T12:04:45.626973324Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:49.550752302Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-22T12:04:50.625966152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:53.703899367Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
level=warn ts=2026-04-22T12:04:55.626764856Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:04:57.857617823Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
level=debug ts=2026-04-22T12:04:57.923889554Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-22T12:05:00.171571276Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:05:00.626792682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:05:00.626817896Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:05:00.626825414Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:05:00.626843062Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:02.012291302Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=warn ts=2026-04-22T12:05:05.626880768Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:06.165628034Z caller=checkpoint.go:378 msg="writing series" size="400 kB"
level=debug ts=2026-04-22T12:05:06.16565149Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=14
level=debug ts=2026-04-22T12:05:09.151973231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:05:10.319284628Z caller=checkpoint.go:378 msg="writing series" size="2.8 kB"
level=warn ts=2026-04-22T12:05:10.626727512Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:14.473230733Z caller=checkpoint.go:378 msg="writing series" size="478 kB"
level=warn ts=2026-04-22T12:05:15.626511134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:18.627407224Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=info ts=2026-04-22T12:05:20.443955231Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:05:20.452858471Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:05:20.626834131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:22.780928626Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T12:05:25.626921955Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:26.935479001Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=info ts=2026-04-22T12:05:27.94225708Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.3 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:05:27.949024027Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T12:05:30.173457402Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T12:05:30.610114317Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:05:30.61118509Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-22T12:05:30.626518053Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:05:30.626537266Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:05:30.626546218Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:05:30.626576682Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:31.089089023Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-22T12:05:35.24305554Z caller=checkpoint.go:378 msg="writing series" size="580 kB"
level=debug ts=2026-04-22T12:05:35.24308291Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-04-22T12:05:35.62669804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:05:36.872539115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49242"
level=debug ts=2026-04-22T12:05:39.396577731Z caller=checkpoint.go:378 msg="writing series" size="206 kB"
level=warn ts=2026-04-22T12:05:40.626802883Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:41.031411765Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:05:43.550284723Z caller=checkpoint.go:378 msg="writing series" size="8.6 kB"
level=warn ts=2026-04-22T12:05:45.626322223Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:47.704420494Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T12:05:50.626385089Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:50.785515429Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T12:05:50.787301765Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T12:05:51.858224973Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=warn ts=2026-04-22T12:05:55.626202924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:05:56.012013633Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=debug ts=2026-04-22T12:06:00.165441538Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
ts=2026-04-22T12:06:00.176041934Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:06:00.626014436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:06:00.626040213Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:06:00.626049105Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:06:00.626052617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:06:00.66138248Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53354"
level=debug ts=2026-04-22T12:06:04.320791792Z caller=checkpoint.go:378 msg="writing series" size="1.6 MB"
level=debug ts=2026-04-22T12:06:04.320815889Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=7
level=warn ts=2026-04-22T12:06:05.62656233Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:08.473657863Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
level=warn ts=2026-04-22T12:06:10.626101594Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:06:12.029454663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.3 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:06:12.037335248Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:06:12.037355705Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T12:06:12.037380687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.3 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:06:12.042806055Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:06:12.465924457Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:06:12.627606838Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-22T12:06:15.188198256Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:06:15.195265879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:06:15.626223099Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:16.78117927Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=warn ts=2026-04-22T12:06:20.626118837Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:20.935300087Z caller=checkpoint.go:378 msg="writing series" size="355 kB"
level=info ts=2026-04-22T12:06:22.777263656Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.3 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:06:22.785048704Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:06:25.094416974Z caller=checkpoint.go:378 msg="writing series" size="8.9 MB"
level=debug ts=2026-04-22T12:06:25.094440181Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.4 MB" series=5
level=warn ts=2026-04-22T12:06:25.626766078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:26.855267852Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T12:06:29.242340215Z caller=checkpoint.go:378 msg="writing series" size="147 kB"
ts=2026-04-22T12:06:30.177905516Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:06:30.610307027Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:06:30.61031289Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:06:30.626816119Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:06:30.626835857Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:06:30.626843868Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:06:30.626856551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:33.396849789Z caller=checkpoint.go:378 msg="writing series" size="242 kB"
level=warn ts=2026-04-22T12:06:35.6269073Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:37.550818188Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=warn ts=2026-04-22T12:06:40.626388584Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:41.703880661Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T12:06:45.626725207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:45.858079269Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-22T12:06:48.25778231Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:06:50.011610426Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=warn ts=2026-04-22T12:06:50.626089454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:54.166229614Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=info ts=2026-04-22T12:06:54.584264935Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:06:54.593707559Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:06:55.625953677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:06:58.323951156Z caller=checkpoint.go:378 msg="writing series" size="8.4 MB"
level=debug ts=2026-04-22T12:06:58.323979301Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.2 MB" series=8
ts=2026-04-22T12:07:00.179714158Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:07:00.626924355Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:07:00.626951118Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:07:00.626958679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:07:00.626964712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:02.473818358Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=info ts=2026-04-22T12:07:03.814625679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="5.6 MB" avg_comp="2.8 MB" total_uncomp="47 MB" avg_uncomp="24 MB" full=2 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:07:03.821824879Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:07:05.626971911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:06.6276781Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-22T12:07:10.626482362Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:10.781291425Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=debug ts=2026-04-22T12:07:14.934829959Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=debug ts=2026-04-22T12:07:14.947868246Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:07:15.626425559Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:19.088711938Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
level=debug ts=2026-04-22T12:07:20.30796556Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:07:20.625970309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:23.243176245Z caller=checkpoint.go:378 msg="writing series" size="128 kB"
level=debug ts=2026-04-22T12:07:23.243227671Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T12:07:25.627058114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:27.396716948Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
ts=2026-04-22T12:07:30.181857399Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:07:30.61107626Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:07:30.611548401Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:07:30.626817971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:07:30.626836017Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:07:30.626843916Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:07:30.626842044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:07:30.665852821Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:49494"
level=debug ts=2026-04-22T12:07:31.550228431Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=warn ts=2026-04-22T12:07:35.626838385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:35.704342606Z caller=checkpoint.go:378 msg="writing series" size="373 kB"
ts=2026-04-22T12:07:36.879187414Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:49652"
level=info ts=2026-04-22T12:07:38.55111999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:07:38.557725373Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T12:07:39.176657275Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43164"
level=debug ts=2026-04-22T12:07:39.858203992Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=warn ts=2026-04-22T12:07:40.626782416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:44.011784308Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=warn ts=2026-04-22T12:07:45.626895659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:07:47.745547637Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:07:47.752055255Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:07:47.752076378Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T12:07:47.752100978Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:07:47.757607668Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:07:48.165738296Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-22T12:07:50.626823951Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:50.837199107Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:07:52.319902948Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=warn ts=2026-04-22T12:07:55.62652888Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:07:56.473362595Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T12:08:00.184090216Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=warn ts=2026-04-22T12:08:00.626647174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:08:00.626642773Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:08:00.626670657Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=debug ts=2026-04-22T12:08:00.626664851Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=8
level=info ts=2026-04-22T12:08:00.626680993Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:08:00.630838756Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-22T12:08:00.634188742Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007559049s
level=warn ts=2026-04-22T12:08:05.626978738Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:08:06.880880099Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:35222"
level=info ts=2026-04-22T12:08:07.074768925Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:08:07.088671611Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:08:10.626029659Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T12:08:15.626626448Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:17.27274041Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:08:20.626614729Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:24.489394648Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:08:25.626854617Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:08:30.185338817Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:08:30.610980136Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:08:30.611406142Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:08:30.626803087Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-22T12:08:30.626809554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:08:30.626835493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:08:30.62685796Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:08:30.626866375Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:08:30.626903267Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-22T12:08:30.627066161Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-22T12:08:31.835502892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:08:31.841543361Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:08:34.781661038Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=warn ts=2026-04-22T12:08:35.626071084Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:38.935267177Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T12:08:40.626445456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:43.089001616Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=info ts=2026-04-22T12:08:43.54024821Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:08:43.551459816Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:08:45.627069679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:47.24326537Z caller=checkpoint.go:378 msg="writing series" size="544 kB"
level=warn ts=2026-04-22T12:08:50.62666378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:51.396646294Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T12:08:55.551230835Z caller=checkpoint.go:378 msg="writing series" size="194 kB"
level=warn ts=2026-04-22T12:08:55.626969061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:08:58.756113638Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:08:59.704825289Z caller=checkpoint.go:378 msg="writing series" size="515 B"
ts=2026-04-22T12:09:00.186094882Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:09:00.6266971Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:09:00.62672133Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:09:00.62673368Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:09:00.626727922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:03.857981302Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T12:09:05.626336711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:08.012326174Z caller=checkpoint.go:378 msg="writing series" size="306 kB"
level=debug ts=2026-04-22T12:09:08.012351527Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-04-22T12:09:10.627008552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:12.16625314Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=warn ts=2026-04-22T12:09:15.626171429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:15.659055055Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-22T12:09:16.187077803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:09:16.192839059Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:09:16.320436489Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-04-22T12:09:20.474081899Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T12:09:20.626688175Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:24.627597428Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T12:09:25.626751636Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:28.741963993Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:09:28.781112582Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
ts=2026-04-22T12:09:30.187687169Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:09:30.610983303Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:09:30.611439403Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:09:30.626829852Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:09:30.626849371Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:09:30.626858008Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:09:30.626876611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:09:30.67199108Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46176"
level=debug ts=2026-04-22T12:09:32.93501216Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=warn ts=2026-04-22T12:09:35.625972107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:37.089123889Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T12:09:38.717859047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:09:38.725393938Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:09:40.626422677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:41.243450173Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=debug ts=2026-04-22T12:09:45.396690903Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T12:09:45.626692685Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:49.55090913Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
level=warn ts=2026-04-22T12:09:50.626132646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:50.85573504Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T12:09:50.857342767Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-22T12:09:53.704106456Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=warn ts=2026-04-22T12:09:55.626666613Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:09:57.858503731Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
level=debug ts=2026-04-22T12:09:59.98173552Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T12:10:00.189904069Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:10:00.626326864Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:10:00.626354681Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T12:10:00.626357158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:10:00.626372822Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:10:02.01262391Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=debug ts=2026-04-22T12:10:02.012647949Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-22T12:10:05.626359363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:06.166063776Z caller=checkpoint.go:378 msg="writing series" size="302 kB"
level=info ts=2026-04-22T12:10:07.013456916Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:10:07.021498485Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:10:10.320346143Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=warn ts=2026-04-22T12:10:10.626152235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:14.47343655Z caller=checkpoint.go:378 msg="writing series" size="552 kB"
level=warn ts=2026-04-22T12:10:15.626905381Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:18.627136252Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T12:10:20.626763688Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:22.781404737Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=warn ts=2026-04-22T12:10:25.626181211Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:26.560334713Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T12:10:26.935700311Z caller=checkpoint.go:378 msg="writing series" size="158 kB"
level=debug ts=2026-04-22T12:10:26.935726528Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
ts=2026-04-22T12:10:30.191968176Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:10:30.610247807Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:10:30.610638296Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:10:30.625984982Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:10:30.626004832Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:10:30.626015396Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:10:30.62601718Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:10:30.675571527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:51202"
level=debug ts=2026-04-22T12:10:31.088919592Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-04-22T12:10:33.297782013Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="8.2 MB" avg_comp="2.7 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=3 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:10:33.306145867Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:10:35.242786846Z caller=checkpoint.go:378 msg="writing series" size="672 kB"
level=debug ts=2026-04-22T12:10:35.345790381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:10:35.626399366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:10:39.187940408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:43864"
level=debug ts=2026-04-22T12:10:39.396875373Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=debug ts=2026-04-22T12:10:39.396899685Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-22T12:10:40.626684179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:43.550990672Z caller=checkpoint.go:378 msg="writing series" size="9.5 kB"
level=warn ts=2026-04-22T12:10:45.626675487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:47.704252742Z caller=checkpoint.go:378 msg="writing series" size="46 kB"
level=warn ts=2026-04-22T12:10:50.626727305Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:51.858198048Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=info ts=2026-04-22T12:10:55.413264115Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:10:55.42285387Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:10:55.626954743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:10:56.012733191Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
level=debug ts=2026-04-22T12:11:00.166579966Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
ts=2026-04-22T12:11:00.193732298Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:11:00.62594115Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:11:00.625968957Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:11:00.625978104Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:11:00.625965557Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:04.322003486Z caller=checkpoint.go:378 msg="writing series" size="1.7 MB"
level=debug ts=2026-04-22T12:11:04.32202831Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.4 MB" series=6
level=warn ts=2026-04-22T12:11:05.626284621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:11:06.892249141Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:34860"
level=debug ts=2026-04-22T12:11:08.474152012Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
ts=2026-04-22T12:11:09.190240404Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:45086"
level=debug ts=2026-04-22T12:11:10.457058766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:11:10.626401278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:12.627658937Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-22T12:11:15.626926988Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:16.78184331Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-22T12:11:19.541642411Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:11:20.626743996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:20.935821558Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=debug ts=2026-04-22T12:11:25.099986812Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-22T12:11:25.10001336Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=5
level=warn ts=2026-04-22T12:11:25.626741548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:29.243183277Z caller=checkpoint.go:378 msg="writing series" size="193 kB"
level=info ts=2026-04-22T12:11:29.706660308Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:11:29.714339953Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T12:11:30.195110268Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:11:30.610259116Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:11:30.6106883Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:11:30.626015865Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:11:30.626033167Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:11:30.626040879Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:11:30.626078554Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:33.39673876Z caller=checkpoint.go:378 msg="writing series" size="260 kB"
level=info ts=2026-04-22T12:11:35.43973221Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=3 total_comp="8.4 MB" avg_comp="2.8 MB" total_uncomp="70 MB" avg_uncomp="23 MB" full=2 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:11:35.446719844Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:11:35.62699857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:11:36.893095281Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:53858"
level=debug ts=2026-04-22T12:11:37.550281893Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
ts=2026-04-22T12:11:39.192396363Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:51918"
level=warn ts=2026-04-22T12:11:40.626125629Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:41.70441245Z caller=checkpoint.go:378 msg="writing series" size="99 kB"
level=debug ts=2026-04-22T12:11:43.483753878Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:11:45.626596825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:45.858658619Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=debug ts=2026-04-22T12:11:50.012545013Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-22T12:11:50.626140774Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:54.16625252Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=debug ts=2026-04-22T12:11:55.426501776Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:11:55.626865309Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:11:58.325601927Z caller=checkpoint.go:378 msg="writing series" size="9.0 MB"
level=debug ts=2026-04-22T12:11:58.325626494Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.9 MB" series=8
ts=2026-04-22T12:12:00.197434609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=warn ts=2026-04-22T12:12:00.626016934Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:12:00.626046055Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:12:00.626075554Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:12:00.626090677Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-22T12:12:00.681666438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:46804"
level=debug ts=2026-04-22T12:12:02.473570082Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
level=warn ts=2026-04-22T12:12:05.626932243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:06.627731606Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-22T12:12:08.074321583Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:12:10.626752502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:10.781834966Z caller=checkpoint.go:378 msg="writing series" size="267 kB"
level=debug ts=2026-04-22T12:12:14.935277558Z caller=checkpoint.go:378 msg="writing series" size="235 kB"
level=warn ts=2026-04-22T12:12:15.626756501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:19.08965545Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=warn ts=2026-04-22T12:12:20.626258406Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:12:23.163657356Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:12:23.171161126Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:12:23.243512705Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=debug ts=2026-04-22T12:12:23.243527194Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T12:12:25.626994326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:27.397058786Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-22T12:12:28.338777774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:12:28.345501269Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:12:28.345521103Z caller=flush.go:240 component=ingester loop=31 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T12:12:28.345545774Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:12:28.352623532Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T12:12:30.19948306Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:12:30.610334861Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:12:30.610335093Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T12:12:30.626854836Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:12:30.626855738Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:12:30.626871048Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:12:30.626878675Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:12:31.55049944Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-22T12:12:35.132190311Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:12:35.146038751Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:12:35.626243076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:35.704677979Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
ts=2026-04-22T12:12:36.896113566Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:45960"
level=info ts=2026-04-22T12:12:38.945261506Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:12:38.954168967Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:12:39.858640934Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=warn ts=2026-04-22T12:12:40.626031676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:44.011980016Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=warn ts=2026-04-22T12:12:45.626747345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:48.16599593Z caller=checkpoint.go:378 msg="writing series" size="89 kB"
level=warn ts=2026-04-22T12:12:50.626677358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:52.320649761Z caller=checkpoint.go:378 msg="writing series" size="449 kB"
level=debug ts=2026-04-22T12:12:52.320673961Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=warn ts=2026-04-22T12:12:55.626613101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:12:56.473963204Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T12:13:00.202068199Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:13:00.626916393Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:13:00.626945489Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:13:00.626953515Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:13:00.626958164Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:00.628029985Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 kB" series=1
level=info ts=2026-04-22T12:13:00.630259802Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-22T12:13:00.633711958Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.006699744s
level=info ts=2026-04-22T12:13:02.856623824Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:13:02.863537703Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:13:02.99688781Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:13:05.626473469Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:13:06.897713394Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:50358"
level=info ts=2026-04-22T12:13:09.4100177Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:13:09.417767053Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:13:10.626464767Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:12.705118591Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T12:13:12.706766981Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-22T12:13:15.626622772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T12:13:20.626260158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-22T12:13:25.626088736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:28.988463286Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-22T12:13:30.203775583Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:13:30.610797083Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:13:30.611220644Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T12:13:30.626524597Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:13:30.626529999Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T12:13:30.626539702Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:13:30.62655727Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:13:30.626566172Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-22T12:13:30.626650945Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-22T12:13:30.626826867Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-22T12:13:34.781609765Z caller=checkpoint.go:378 msg="writing series" size="217 kB"
level=warn ts=2026-04-22T12:13:35.626410596Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:38.935375626Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
ts=2026-04-22T12:13:39.19918511Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:56756"
level=warn ts=2026-04-22T12:13:40.626161327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:43.088498824Z caller=checkpoint.go:378 msg="writing series" size="15 kB"
level=warn ts=2026-04-22T12:13:45.626925372Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:47.243342714Z caller=checkpoint.go:378 msg="writing series" size="663 kB"
level=warn ts=2026-04-22T12:13:50.626384899Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:13:51.396377452Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=debug ts=2026-04-22T12:13:55.550951309Z caller=checkpoint.go:378 msg="writing series" size="218 kB"
level=debug ts=2026-04-22T12:13:55.550974425Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-22T12:13:55.626910247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:13:56.203094378Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:13:56.209108502Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:13:57.678554487Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T12:13:58.457815696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:13:59.70477971Z caller=checkpoint.go:378 msg="writing series" size="643 B"
ts=2026-04-22T12:14:00.205548224Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:14:00.57502594Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:14:00.581781718Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=info ts=2026-04-22T12:14:00.626798116Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:14:00.626826507Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:14:00.626841607Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:14:00.62684947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:14:00.688090689Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:58540"
level=debug ts=2026-04-22T12:14:03.858525812Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
level=warn ts=2026-04-22T12:14:05.626431478Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:14:06.900927071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:33330"
level=debug ts=2026-04-22T12:14:08.011934032Z caller=checkpoint.go:378 msg="writing series" size="424 kB"
level=warn ts=2026-04-22T12:14:10.626112119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:12.166165864Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-22T12:14:15.62601267Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:16.319748014Z caller=checkpoint.go:378 msg="writing series" size="55 kB"
level=debug ts=2026-04-22T12:14:20.473803608Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-22T12:14:20.626281781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:23.59279764Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:14:24.627289046Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T12:14:25.626341057Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:28.781686109Z caller=checkpoint.go:378 msg="writing series" size="56 kB"
ts=2026-04-22T12:14:30.207663527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-a0952cc5 10.128.0.74:7946"
level=info ts=2026-04-22T12:14:30.610268898Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:14:30.611854057Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:14:30.626112613Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:14:30.626131865Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:14:30.626142277Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:14:30.626142502Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:32.934816956Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-22T12:14:35.626928833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:37.089078119Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=info ts=2026-04-22T12:14:38.438926239Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:14:38.447640966Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:14:38.4476599Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-22T12:14:38.447685513Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:14:38.453702926Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:14:40.626075051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:41.242804524Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-22T12:14:45.396912224Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-22T12:14:45.626305182Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:49.550741858Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
level=debug ts=2026-04-22T12:14:49.550765049Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=debug ts=2026-04-22T12:14:49.804248587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:14:50.626241534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:14:51.592466052Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:14:51.599312625Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:14:53.704891529Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
level=warn ts=2026-04-22T12:14:55.626260725Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:14:57.858467107Z caller=checkpoint.go:378 msg="writing series" size="9.6 kB"
level=debug ts=2026-04-22T12:14:57.890653971Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T12:14:57.890709299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:14:57.895395765Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-22T12:14:58.083932769Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:14:58.089926736Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T12:15:00.209305115Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:15:00.62659923Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:15:00.626625382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:15:00.626639497Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:15:00.626656459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:02.012017517Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=warn ts=2026-04-22T12:15:05.625970998Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:06.166075285Z caller=checkpoint.go:378 msg="writing series" size="439 kB"
level=debug ts=2026-04-22T12:15:10.319987772Z caller=checkpoint.go:378 msg="writing series" size="3.3 kB"
level=warn ts=2026-04-22T12:15:10.626684867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:14.473945459Z caller=checkpoint.go:378 msg="writing series" size="561 kB"
level=debug ts=2026-04-22T12:15:14.473970097Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-22T12:15:15.626910699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:15:16.233162241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:15:16.238790226Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-22T12:15:18.627637207Z caller=checkpoint.go:378 msg="writing series" size="7.1 kB"
level=warn ts=2026-04-22T12:15:20.626755772Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:22.781510694Z caller=checkpoint.go:378 msg="writing series" size="87 kB"
level=info ts=2026-04-22T12:15:24.154321679Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:15:24.162511737Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-22T12:15:25.626454622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:26.934808035Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
ts=2026-04-22T12:15:30.211802975Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:15:30.610270368Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:15:30.61069617Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=debug ts=2026-04-22T12:15:30.611934092Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-22T12:15:30.611976037Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974288 n=1
level=debug ts=2026-04-22T12:15:30.612016763Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974288 n=1
level=info ts=2026-04-22T12:15:30.625911074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:15:30.625923828Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:15:30.625931455Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:15:30.625946924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:30.645321606Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:15:31.088677484Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=debug ts=2026-04-22T12:15:35.243341992Z caller=checkpoint.go:378 msg="writing series" size="775 kB"
level=debug ts=2026-04-22T12:15:35.243367881Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-22T12:15:35.626248358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:39.397146107Z caller=checkpoint.go:378 msg="writing series" size="314 kB"
level=warn ts=2026-04-22T12:15:40.626409968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:40.733364204Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-22T12:15:43.550722517Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-22T12:15:45.626107194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:15:47.379503722Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:15:47.387530595Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:15:47.704715194Z caller=checkpoint.go:378 msg="writing series" size="47 kB"
level=warn ts=2026-04-22T12:15:50.626758739Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:51.858558604Z caller=checkpoint.go:378 msg="writing series" size="216 kB"
level=info ts=2026-04-22T12:15:54.025575921Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:15:54.031493918Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T12:15:55.626361239Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:15:56.011543156Z caller=checkpoint.go:378 msg="writing series" size="38 kB"
level=debug ts=2026-04-22T12:15:58.277377112Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:16:00.165744404Z caller=checkpoint.go:378 msg="writing series" size="353 kB"
ts=2026-04-22T12:16:00.213520671Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:16:00.626325765Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:16:00.62635313Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:16:00.626362731Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:16:00.626362472Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:04.320896697Z caller=checkpoint.go:378 msg="writing series" size="1.8 MB"
level=debug ts=2026-04-22T12:16:04.320917147Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.7 MB" series=7
level=warn ts=2026-04-22T12:16:05.626425807Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:16:06.38289016Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:16:06.39156223Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T12:16:06.908918379Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:51460"
level=debug ts=2026-04-22T12:16:08.473497523Z caller=checkpoint.go:378 msg="writing series" size="2.9 kB"
ts=2026-04-22T12:16:09.210321329Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:42458"
level=warn ts=2026-04-22T12:16:10.626366953Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:12.627110289Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-22T12:16:15.626570525Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:16.781462518Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=warn ts=2026-04-22T12:16:20.626402135Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:20.93525395Z caller=checkpoint.go:378 msg="writing series" size="400 kB"
level=debug ts=2026-04-22T12:16:25.097626669Z caller=checkpoint.go:378 msg="writing series" size="13 MB"
level=debug ts=2026-04-22T12:16:25.097649907Z caller=checkpoint.go:390 msg="flushing series" totalSize="13 MB" series=5
level=warn ts=2026-04-22T12:16:25.625998339Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:29.243118591Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
ts=2026-04-22T12:16:30.216071494Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=info ts=2026-04-22T12:16:30.610315765Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:16:30.61180899Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:16:30.625962522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-22T12:16:30.625972531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:30.626000606Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:16:30.626009029Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:16:32.871807833Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-22T12:16:33.396164669Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-22T12:16:35.626712Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-22T12:16:36.909968673Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:48998"
level=debug ts=2026-04-22T12:16:37.550058366Z caller=checkpoint.go:378 msg="writing series" size="7.7 kB"
level=debug ts=2026-04-22T12:16:40.040124377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:16:40.62631373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:41.704840049Z caller=checkpoint.go:378 msg="writing series" size="107 kB"
level=info ts=2026-04-22T12:16:45.104630018Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:16:45.109623404Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T12:16:45.626195547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:16:45.842271291Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:16:45.848865058Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=debug ts=2026-04-22T12:16:45.85836754Z caller=checkpoint.go:378 msg="writing series" size="246 kB"
level=debug ts=2026-04-22T12:16:50.012343534Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
level=warn ts=2026-04-22T12:16:50.626662289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:16:50.737357555Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:16:50.743621675Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:16:54.165472686Z caller=checkpoint.go:378 msg="writing series" size="112 kB"
level=warn ts=2026-04-22T12:16:55.626384454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:16:58.325648285Z caller=checkpoint.go:378 msg="writing series" size="9.7 MB"
level=debug ts=2026-04-22T12:16:58.325676774Z caller=checkpoint.go:390 msg="flushing series" totalSize="10 MB" series=8
ts=2026-04-22T12:17:00.21790582Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T12:17:00.274963509Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T12:17:00.626801222Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:17:00.626828075Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:17:00.626837239Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:17:00.626842136Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:02.473328584Z caller=checkpoint.go:378 msg="writing series" size="180 kB"
level=warn ts=2026-04-22T12:17:05.626123897Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:06.627334943Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
ts=2026-04-22T12:17:06.91119609Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:46702"
ts=2026-04-22T12:17:09.214831339Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:36886"
level=warn ts=2026-04-22T12:17:10.625987462Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:10.78140744Z caller=checkpoint.go:378 msg="writing series" size="324 kB"
level=debug ts=2026-04-22T12:17:14.935494489Z caller=checkpoint.go:378 msg="writing series" size="90 kB"
level=warn ts=2026-04-22T12:17:15.626016238Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:19.088818824Z caller=checkpoint.go:378 msg="writing series" size="472 kB"
level=debug ts=2026-04-22T12:17:19.088844753Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-22T12:17:20.62617371Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:23.243249162Z caller=checkpoint.go:378 msg="writing series" size="228 kB"
level=debug ts=2026-04-22T12:17:24.589401598Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T12:17:24.589465386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="325 kB" avg_comp="325 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T12:17:24.593872998Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db4f82c91%3A19db518c874%3A31cc8427?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-22T12:17:25.626433228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:27.39673542Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-22T12:17:29.94444934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:17:29.950584781Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
ts=2026-04-22T12:17:30.220063159Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-8lrvh-3c08d562 10.129.0.81:7946"
level=debug ts=2026-04-22T12:17:30.315723385Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-22T12:17:30.317180749Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=info ts=2026-04-22T12:17:30.610265215Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:17:30.610774819Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=warn ts=2026-04-22T12:17:30.625966634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:17:30.625972338Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:17:30.625996582Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:17:30.626003957Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:17:31.550903248Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-22T12:17:32.026440644Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:17:32.031147824Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T12:17:35.626737152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:35.704717631Z caller=checkpoint.go:378 msg="writing series" size="545 kB"
ts=2026-04-22T12:17:39.216658533Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:39958"
level=debug ts=2026-04-22T12:17:39.858663903Z caller=checkpoint.go:378 msg="writing series" size="190 kB"
level=warn ts=2026-04-22T12:17:40.626374279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:17:41.215349282Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="329 kB" avg_comp="329 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T12:17:41.222320605Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db4f82c91%3A19db518c874%3A31cc8427?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-22T12:17:42.607321321Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:17:42.613975025Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-22T12:17:44.011761469Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
level=warn ts=2026-04-22T12:17:45.626279989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:48.165459295Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=debug ts=2026-04-22T12:17:48.165488083Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=warn ts=2026-04-22T12:17:50.626061743Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:52.320357827Z caller=checkpoint.go:378 msg="writing series" size="309 kB"
level=debug ts=2026-04-22T12:17:52.710435328Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-22T12:17:55.626113846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:17:56.47351249Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
ts=2026-04-22T12:18:00.222621056Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:18:00.626203403Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:18:00.626240548Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-22T12:18:00.626234848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:18:00.626248517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-22T12:18:00.627310691Z caller=checkpoint.go:390 msg="flushing series" totalSize="320 kB" series=2
level=info ts=2026-04-22T12:18:00.633512769Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-22T12:18:00.637193318Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.010404908s
ts=2026-04-22T12:18:00.704184116Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:38946"
level=warn ts=2026-04-22T12:18:05.626325714Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:18:05.687164089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="329 kB" avg_comp="329 kB" total_uncomp="3.3 MB" avg_uncomp="3.3 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-22T12:18:05.691682511Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19db4f82c91%3A19db518c874%3A31cc8427?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-22T12:18:06.914768027Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.74:57830"
ts=2026-04-22T12:18:09.219197247Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.83:33310"
level=warn ts=2026-04-22T12:18:10.626564352Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:18:13.190789392Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-22T12:18:15.626646752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:18:18.245544502Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.7 MB" avg_comp="1.7 MB" total_uncomp="10 MB" avg_uncomp="10 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-22T12:18:18.251731839Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19db4f772cb%3A19db51d3820%3Ad38be368?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-22T12:18:20.626091302Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:18:23.643747492Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=123a594a6c15ca1f immediate=false num_chunks=4 total_comp="11 MB" avg_comp="2.8 MB" total_uncomp="95 MB" avg_uncomp="24 MB" full=3 synced=1 labels="{stream=\"openshift-pipelines\"}"
level=error ts=2026-04-22T12:18:23.650686355Z caller=flush.go:262 component=ingester loop=31 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/123a594a6c15ca1f/19db4f7dc32%3A19db5094b9c%3A803a5f58?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-pipelines\"}"
level=warn ts=2026-04-22T12:18:25.626113922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:18:26.815556809Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-22T12:18:26.837629317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="8.8 MB" avg_comp="2.2 MB" total_uncomp="75 MB" avg_uncomp="19 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-22T12:18:26.844307917Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19db4f771b4%3A19db4f82c82%3A826776da?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-22T12:18:30.22435403Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-hp44f-6a563897 10.129.0.83:7946"
level=info ts=2026-04-22T12:18:30.610249863Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-22T12:18:30.61054927Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.129.0.84:11211 count=1
level=info ts=2026-04-22T12:18:30.626850309Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-22T12:18:30.626857441Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-22T12:18:30.626875565Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-22T12:18:30.626882718Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-22T12:18:30.62688467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-22T12:18:30.626976536Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-22T12:18:30.627149208Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
ts=2026-04-22T12:18:30.705992935Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.129.0.81:53354"
level=debug ts=2026-04-22T12:18:34.781424817Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-22T12:18:35.626890527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:18:38.935714536Z caller=checkpoint.go:378 msg="writing series" size="3.9 kB"
level=warn ts=2026-04-22T12:18:40.626224538Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-22T12:18:43.08873817Z caller=checkpoint.go:378 msg="writing series" size="15 kB"