level=info ts=2026-04-28T13:04:42.613537367Z caller=main.go:127 msg="Starting Loki" version="(version=3.6.3, branch=release-3.6.x, revision=9385bc63)"
level=info ts=2026-04-28T13:04:42.613574396Z caller=main.go:128 msg="Loading configuration file" filename=/etc/loki/config/config.yaml
level=debug ts=2026-04-28T13:04:42.614019867Z caller=modules.go:1984 msg="initializing ingester query tags interceptors"
level=info ts=2026-04-28T13:04:42.614833964Z caller=server.go:386 msg="server listening on addresses" http=[::]:3100 grpc=[::]:9095
level=debug ts=2026-04-28T13:04:42.615297194Z caller=netutil.go:85 msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T13:04:42.615374446Z caller=netutil.go:102 msg="detected highest quality address" ipAddr=10.128.0.74 inf=eth0
level=info ts=2026-04-28T13:04:42.616237424Z caller=memberlist_client.go:484 msg="Using memberlist cluster label and node name" cluster_label= node=loki-ingester-0-358faf8e
ts=2026-04-28T13:04:42.616321087Z caller=memberlist_logger.go:74 level=debug msg="configured Transport is not a NodeAwareTransport and some features may not work as desired"
level=debug ts=2026-04-28T13:04:42.616336262Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.74 advertisePort=7946
level=debug ts=2026-04-28T13:04:42.616372442Z caller=tcp_transport.go:435 component="memberlist TCPTransport" msg=FinalAdvertiseAddr advertiseAddr=10.128.0.74 advertisePort=7946
level=info ts=2026-04-28T13:04:42.616481913Z caller=memberlist_client.go:628 msg="memberlist fast-join starting" nodes_found=1 to_join=4
ts=2026-04-28T13:04:42.620137247Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=info ts=2026-04-28T13:04:42.62015603Z caller=memberlist_client.go:634 msg="fast-joining node failed" node=loki-memberlist.product-kubearchive-logging.svc.cluster.local err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T13:04:42.620167545Z caller=memberlist_client.go:644 msg="memberlist fast-join failed because no node has been successfully reached" elapsed_time=3.687408ms
level=error ts=2026-04-28T13:04:42.620177582Z caller=memberlist_client.go:521 msg="failed to fast-join the memberlist cluster at startup" err="no memberlist node reached during fast-join procedure"
level=info ts=2026-04-28T13:04:42.620192995Z caller=memberlist_client.go:666 phase=startup msg="joining memberlist cluster" join_members=loki-memberlist.product-kubearchive-logging.svc.cluster.local
level=error ts=2026-04-28T13:04:42.620901173Z caller=resolver.go:100 msg="failed to lookup SRV records" host=_memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local err="lookup _memcached-client._tcp.loki-chunks-cache.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:04:42.620923363Z caller=memcached_client_selector.go:109 msg="updating servers" servers= count=0
level=info ts=2026-04-28T13:04:42.622340624Z caller=shipper.go:165 index-store=tsdb-2024-04-01 msg="starting index shipper in WO mode"
level=info ts=2026-04-28T13:04:42.622810054Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-28T13:04:42.622823967Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-28T13:04:42.62282231Z
level=debug ts=2026-04-28T13:04:42.622841398Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
level=info ts=2026-04-28T13:04:42.622862712Z caller=manager.go:86 index-store=tsdb-2024-04-01 component=tsdb-manager msg="loaded leftover local indices" err=null successful=true buckets=0 indices=0 failures=0
level=info ts=2026-04-28T13:04:42.622893187Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:04:42.622916228Z caller=head_manager.go:313 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="loaded wals by period" groups=0
level=debug ts=2026-04-28T13:04:42.62292587Z caller=manager.go:263 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building WALs" n=0 ts=2026-04-28T13:04:42.622924463Z
level=debug ts=2026-04-28T13:04:42.62293276Z caller=manager.go:287 index-store=tsdb-2024-04-01 component=tsdb-manager msg="recovering tenant heads"
ts=2026-04-28T13:04:42.623736295Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:04:42.623751252Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=1 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:04:42.624579574Z caller=netutil.go:85 component=ingester msg="looking for addresses" inf="[eth0 lo]" inet6enabled=false
level=debug ts=2026-04-28T13:04:42.624677078Z caller=netutil.go:102 component=ingester msg="detected highest quality address" ipAddr=10.128.0.74 inf=eth0
level=debug ts=2026-04-28T13:04:42.637471921Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=runtime-config
level=debug ts=2026-04-28T13:04:42.637536704Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=store
level=debug ts=2026-04-28T13:04:42.637551704Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=memberlist-kv
level=info ts=2026-04-28T13:04:42.637546864Z caller=module_service.go:82 msg=starting module=runtime-config
level=info ts=2026-04-28T13:04:42.63756816Z caller=module_service.go:82 msg=starting module=server
level=debug ts=2026-04-28T13:04:42.637572092Z caller=module_service.go:72 msg="module waiting for initialization" module=memberlist-kv waiting_for=server
level=debug ts=2026-04-28T13:04:42.637581837Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=memberlist-kv
level=info ts=2026-04-28T13:04:42.637593661Z caller=module_service.go:82 msg=starting module=memberlist-kv
level=debug ts=2026-04-28T13:04:42.637604599Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=ring
level=debug ts=2026-04-28T13:04:42.637608985Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=runtime-config
level=debug ts=2026-04-28T13:04:42.637883973Z caller=module_service.go:72 msg="module waiting for initialization" module=ring waiting_for=server
level=info ts=2026-04-28T13:04:42.637895278Z caller=module_service.go:82 msg=starting module=ring
level=info ts=2026-04-28T13:04:42.63791728Z caller=ring.go:365 msg="ring doesn't exist in KV store yet"
level=debug ts=2026-04-28T13:04:42.637940972Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=server
level=debug ts=2026-04-28T13:04:42.637945755Z caller=module_service.go:72 msg="module waiting for initialization" module=store waiting_for=memberlist-kv
level=info ts=2026-04-28T13:04:42.637948764Z caller=module_service.go:82 msg=starting module=store
level=debug ts=2026-04-28T13:04:42.637949423Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=runtime-config
level=debug ts=2026-04-28T13:04:42.637955564Z caller=module_service.go:72 msg="module waiting for initialization" module=analytics waiting_for=server
level=debug ts=2026-04-28T13:04:42.637957613Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=analytics
level=info ts=2026-04-28T13:04:42.637960801Z caller=module_service.go:82 msg=starting module=analytics
level=debug ts=2026-04-28T13:04:42.638075646Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=memberlist-kv
level=debug ts=2026-04-28T13:04:42.638078921Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=ring
level=debug ts=2026-04-28T13:04:42.638081979Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=runtime-config
level=debug ts=2026-04-28T13:04:42.638084755Z caller=module_service.go:72 msg="module waiting for initialization" module=ingester waiting_for=server
level=info ts=2026-04-28T13:04:42.638087584Z caller=module_service.go:82 msg=starting module=ingester
level=info ts=2026-04-28T13:04:42.638104501Z caller=ingester.go:565 component=ingester msg="recovering from checkpoint"
level=info ts=2026-04-28T13:04:42.638135088Z caller=recovery.go:42 component=ingester msg="no checkpoint found, treating as no-op"
level=info ts=2026-04-28T13:04:42.638209418Z caller=ingester.go:581 component=ingester msg="recovered WAL checkpoint recovery finished" elapsed=107.094µs errors=false
level=info ts=2026-04-28T13:04:42.638215569Z caller=ingester.go:587 component=ingester msg="recovering from WAL"
level=info ts=2026-04-28T13:04:42.638290163Z caller=ingester.go:603 component=ingester msg="WAL segment recovery finished" elapsed=187.996µs errors=false
level=info ts=2026-04-28T13:04:42.638296551Z caller=ingester.go:551 component=ingester msg="closing recoverer"
level=info ts=2026-04-28T13:04:42.638305188Z caller=ingester.go:559 component=ingester msg="WAL recovery finished" time=202.794µs
level=info ts=2026-04-28T13:04:42.638354705Z caller=wal.go:158 msg=started component=wal
level=info ts=2026-04-28T13:04:42.638584576Z caller=lifecycler.go:687 component=ingester msg="not loading tokens from file, tokens file path is empty"
level=info ts=2026-04-28T13:04:42.638577149Z caller=ingester.go:772 component=ingester msg="sleeping for initial delay before starting periodic flushing" delay=5.624183429s
level=info ts=2026-04-28T13:04:42.638593594Z caller=ingester.go:467 component=ingester msg="autoforget is enabled and will remove unhealthy instances from the ring after 10m0s with no heartbeat"
level=info ts=2026-04-28T13:04:42.638603414Z caller=lifecycler.go:714 component=ingester msg="instance not found in ring, adding with no tokens" ring=ingester
level=info ts=2026-04-28T13:04:42.638600363Z caller=loki.go:599 msg="Loki started" startup_time=90.027346ms
level=debug ts=2026-04-28T13:04:42.638696051Z caller=lifecycler.go:552 component=ingester msg="JoinAfter expired" ring=ingester
level=info ts=2026-04-28T13:04:42.63870183Z caller=lifecycler.go:556 component=ingester msg="auto-joining cluster after timeout" ring=ingester
level=debug ts=2026-04-28T13:04:42.63878032Z caller=memberlist_client.go:1172 msg="CAS attempt failed" err="no change detected" retry=true
ts=2026-04-28T13:04:44.464400939Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:04:44.464426035Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=2 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T13:04:46.51124395Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:04:46.511268605Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=3 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T13:04:47.638828013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:04:52.639430722Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:04:54.133366704Z caller=memberlist_logger.go:74 level=warn msg="Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:04:54.133392945Z caller=memberlist_client.go:700 phase=startup msg="joining memberlist cluster" attempts=4 max_attempts=10 err="1 error occurred:\n\t* Failed to resolve loki-memberlist.product-kubearchive-logging.svc.cluster.local: lookup loki-memberlist.product-kubearchive-logging.svc.cluster.local on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T13:04:57.638902748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:04:57.98243373Z caller=logging.go:137 msg="GET /ready (503) 88.102µs"
level=warn ts=2026-04-28T13:05:02.639623687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:05:07.639267498Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:05:07.98181561Z caller=logging.go:137 msg="GET /ready (503) 65.733µs"
ts=2026-04-28T13:05:09.581726621Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.129.0.86:7946: dial tcp 10.129.0.86:7946: connect: connection refused"
ts=2026-04-28T13:05:09.582599845Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.73:7946"
ts=2026-04-28T13:05:09.583841107Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.128.0.71:7946"
ts=2026-04-28T13:05:09.585917099Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with:  10.130.0.67:7946"
ts=2026-04-28T13:05:09.588750074Z caller=memberlist_logger.go:74 level=debug msg="Failed to join 10.130.0.68:7946: dial tcp 10.130.0.68:7946: connect: connection refused"
level=info ts=2026-04-28T13:05:09.588761417Z caller=memberlist_client.go:673 phase=startup msg="joining memberlist cluster succeeded" reached_nodes=3 elapsed_time=26.968562495s
level=info ts=2026-04-28T13:05:12.639472289Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-28T13:05:12.639500736Z caller=recalculate_owned_streams.go:63 msg="detected ring changes, re-evaluating streams ownership"
level=info ts=2026-04-28T13:05:12.639505551Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:05:12.639521471Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:05:17.639355259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:05:20.815737897Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:05:22.302064472Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49340"
level=warn ts=2026-04-28T13:05:22.638849364Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:05:27.639710221Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:05:32.639448534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:05:36.173688616Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:05:37.638943103Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:05:38.272018273Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:05:42.622241514Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:05:42.623279819Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:05:42.638710029Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:05:42.638726263Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:05:42.638735246Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:05:42.638728873Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:05:46.812939465Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54590"
level=warn ts=2026-04-28T13:05:47.639143326Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:05:51.496108251Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
ts=2026-04-28T13:05:52.302963698Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:42190"
level=warn ts=2026-04-28T13:05:52.639312192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:05:56.714920851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:05:56.7149884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:05:56.724242379Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:05:57.639500819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:06:02.638839268Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:06:06.174990771Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:06:07.472863747Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T13:06:07.639435829Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:06:11.212080203Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:06:11.220531722Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:06:12.638709941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:06:12.638706248Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:06:12.638732684Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:06:12.638740446Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:06:17.638916342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:06:22.304527682Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:35746"
level=warn ts=2026-04-28T13:06:22.639155818Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:06:26.720290346Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:06:27.374112818Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:06:27.638641746Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:06:32.639070083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:06:34.680233901Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:06:34.688219319Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:06:36.176711119Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:06:37.638878235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:06:42.622032391Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:06:42.623099035Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:06:42.639602077Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:06:42.639620355Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:06:42.639628021Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:06:42.639640684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:06:47.638799158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:06:52.639434044Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:06:53.552518603Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T13:06:53.554464082Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:06:57.639162424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:06:58.81228539Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:07:02.639427327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:07:06.178451808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:07:07.638645287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:07:12.639258158Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:07:12.639271911Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:07:12.639290067Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:07:12.639298501Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:07:14.882263188Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34534"
level=warn ts=2026-04-28T13:07:17.639375641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:07:19.337494182Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:07:19.344732822Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:07:22.639227972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:07:27.638768479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:07:28.461869953Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:07:32.012104489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:07:32.6389213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:07:36.180554339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:07:37.639415752Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:07:42.622927482Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:07:42.623345784Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:07:42.639248285Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:07:42.639268539Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:07:42.639275933Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:07:42.639272317Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:07:44.885491966Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37048"
level=warn ts=2026-04-28T13:07:47.63924719Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:07:52.638791263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:07:57.639616295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:08:02.053144963Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:08:02.638991289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:08:06.183598045Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:08:07.63916101Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:08:12.639537163Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:08:12.639548096Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:08:12.639563615Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:08:12.639571488Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:08:17.639538907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:08:19.103441447Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:08:19.110727647Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:08:22.638655656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:08:27.638762408Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:08:30.50470893Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:08:30.972690483Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=warn ts=2026-04-28T13:08:32.639083922Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:08:36.185773896Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:08:37.639406579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:08:42.62297985Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:08:42.623906467Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:08:42.639169166Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:08:42.639188269Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:08:42.639197621Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:08:42.639199619Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:08:47.63938249Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:08:52.638990398Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:08:57.638764287Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:08:59.521761318Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:09:02.63969033Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:09:06.18786999Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:09:07.639591373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:09:11.328519083Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:09:11.335512706Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:09:12.638644961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:09:12.638653875Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:09:12.638690427Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:09:12.638698506Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:09:16.826234094Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45710"
level=warn ts=2026-04-28T13:09:17.639058145Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:09:22.313841575Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37918"
level=warn ts=2026-04-28T13:09:22.638781866Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:09:27.639158161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:09:32.639264864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:09:33.155097719Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:09:36.189839651Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:09:37.63920908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:09:42.62261244Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:09:42.623685989Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:09:42.639135784Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:09:42.639152888Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:09:42.639146574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:09:42.639161221Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:09:42.639160897Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:09:42.63928479Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000000
level=debug ts=2026-04-28T13:09:42.639569566Z caller=checkpoint.go:378 msg="writing series" size="247 kB"
level=debug ts=2026-04-28T13:09:46.995123969Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:09:47.639448479Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:09:48.129514705Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:09:51.350235996Z caller=checkpoint.go:378 msg="writing series" size="286 kB"
level=warn ts=2026-04-28T13:09:52.639142826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:09:55.704073546Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:09:57.639513508Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:00.059380387Z caller=checkpoint.go:378 msg="writing series" size="196 kB"
level=warn ts=2026-04-28T13:10:02.639357658Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:04.413714426Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-28T13:10:05.14810645Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:10:06.191266567Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=info ts=2026-04-28T13:10:06.568877459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:10:06.577151636Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:10:07.638839579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:08.768613515Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-28T13:10:12.638981069Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:10:12.639006644Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:10:12.639014397Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:10:12.639029962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:13.124255436Z caller=checkpoint.go:378 msg="writing series" size="50 kB"
level=debug ts=2026-04-28T13:10:17.478743632Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
level=debug ts=2026-04-28T13:10:17.478769783Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-28T13:10:17.639420247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:21.833486601Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:10:22.638647266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:26.188335503Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
level=warn ts=2026-04-28T13:10:27.639440562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:29.393833766Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:10:30.543167458Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
level=warn ts=2026-04-28T13:10:32.638719914Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:34.897826098Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
ts=2026-04-28T13:10:36.193711192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:10:37.639252225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:39.252677345Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:10:42.622958243Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:10:42.623290629Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:10:42.639101457Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:10:42.639119328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:10:42.639127187Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:10:42.639128213Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:43.608195546Z caller=checkpoint.go:378 msg="writing series" size="582 kB"
ts=2026-04-28T13:10:46.832298769Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39394"
level=warn ts=2026-04-28T13:10:47.63964289Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:47.962582254Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T13:10:50.931044418Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=1 total_comp="2.5 MB" avg_comp="2.5 MB" total_uncomp="24 MB" avg_uncomp="24 MB" full=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:10:50.939176789Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:10:52.31713229Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-28T13:10:52.639632882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:56.677615361Z caller=checkpoint.go:378 msg="writing series" size="5.0 MB"
level=debug ts=2026-04-28T13:10:56.677637364Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.7 MB" series=9
level=warn ts=2026-04-28T13:10:57.639500666Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:10:57.683720286Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:11:00.782244587Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=debug ts=2026-04-28T13:11:01.027086335Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:11:02.638701814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:05.382429068Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
ts=2026-04-28T13:11:06.195888639Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:11:07.639392781Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:09.737162876Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=info ts=2026-04-28T13:11:12.639141411Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:11:12.639166661Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:11:12.639174458Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:11:12.639190314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:14.092128143Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:11:17.639577158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:18.446830861Z caller=checkpoint.go:378 msg="writing series" size="178 kB"
level=warn ts=2026-04-28T13:11:22.638685423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:22.800993537Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:11:26.168833767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:11:27.156560619Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:11:27.638891655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:31.511137726Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:11:32.638639349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:35.865712281Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:11:36.196654376Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:11:37.639036235Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:40.220698648Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:11:42.623094458Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:11:42.623786398Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:11:42.639605294Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:11:42.639611585Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:42.639627245Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:11:42.63963526Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:11:44.575554444Z caller=checkpoint.go:378 msg="writing series" size="267 kB"
level=info ts=2026-04-28T13:11:45.004710163Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:11:45.011345019Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:11:47.639326917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:48.930257925Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-28T13:11:52.321310214Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41522"
level=warn ts=2026-04-28T13:11:52.639480657Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:53.285494747Z caller=checkpoint.go:378 msg="writing series" size="124 kB"
level=warn ts=2026-04-28T13:11:57.639197171Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:11:57.640279648Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:12:01.408304474Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n"
level=error ts=2026-04-28T13:12:01.409861757Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:12:01.667972056Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:12:01.994737643Z caller=checkpoint.go:378 msg="writing series" size="3.6 kB"
level=warn ts=2026-04-28T13:12:02.639556377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:12:06.197940835Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:12:06.349877532Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:12:07.639357017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:10.704964493Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:12:12.639104637Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:12:12.639128444Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:12:12.639136253Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:12:12.639130019Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:12:14.914892119Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:38602"
level=debug ts=2026-04-28T13:12:15.059799838Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:12:17.638802884Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:19.413738485Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=debug ts=2026-04-28T13:12:19.413762213Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=19
level=warn ts=2026-04-28T13:12:22.639571846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:23.768963364Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T13:12:27.639195971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:12:27.881603088Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:12:27.892384131Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:12:27.892406363Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:12:27.892431401Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:12:27.899501979Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:12:28.123721523Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:12:32.480075324Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T13:12:32.480098727Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-28T13:12:32.639366954Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:33.46244358Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:12:36.198997234Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:12:36.833634504Z caller=checkpoint.go:378 msg="writing series" size="49 kB"
level=warn ts=2026-04-28T13:12:37.639083917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:41.187987709Z caller=checkpoint.go:378 msg="writing series" size="213 kB"
level=debug ts=2026-04-28T13:12:42.622173722Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:12:42.623239291Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-28T13:12:42.638705388Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:12:42.638721426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:12:42.638739208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:12:42.638748119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:12:45.051139521Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:12:45.543564605Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
ts=2026-04-28T13:12:46.840547277Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60362"
level=warn ts=2026-04-28T13:12:47.639105107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:12:47.868598538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:12:47.87568405Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:12:49.898248795Z caller=checkpoint.go:378 msg="writing series" size="514 B"
ts=2026-04-28T13:12:52.32465664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41722"
level=warn ts=2026-04-28T13:12:52.639049503Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:54.253224492Z caller=checkpoint.go:378 msg="writing series" size="10 kB"
level=warn ts=2026-04-28T13:12:57.639359138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:12:58.607683203Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-28T13:13:01.094346886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:13:02.639541717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:02.963099138Z caller=checkpoint.go:378 msg="writing series" size="154 kB"
ts=2026-04-28T13:13:06.200331335Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:13:07.317652651Z caller=checkpoint.go:378 msg="writing series" size="72 kB"
level=warn ts=2026-04-28T13:13:07.639246423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:11.67273763Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:13:12.638606672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:13:12.638630563Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:13:12.638639278Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:13:12.638643292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:16.026874715Z caller=checkpoint.go:378 msg="writing series" size="8.7 kB"
ts=2026-04-28T13:13:16.842145988Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:40372"
level=warn ts=2026-04-28T13:13:17.639451936Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:20.3821369Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
ts=2026-04-28T13:13:22.326112785Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45134"
level=warn ts=2026-04-28T13:13:22.639451555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:24.736411403Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-28T13:13:24.958114164Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:13:24.967738079Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:13:27.639374703Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:28.853335886Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:13:29.091944783Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-04-28T13:13:32.639018928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:33.446138601Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:13:36.201524207Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:13:37.639538741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:37.800896104Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T13:13:37.800919787Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
level=debug ts=2026-04-28T13:13:42.156556759Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-28T13:13:42.62293454Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:13:42.623452979Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=warn ts=2026-04-28T13:13:42.639507011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:13:42.639511088Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:13:42.639531382Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:13:42.639538823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:13:46.061443955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:13:46.510673569Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
ts=2026-04-28T13:13:46.843697092Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60172"
level=warn ts=2026-04-28T13:13:47.638643428Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:50.865742231Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-04-28T13:13:52.32746128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:49730"
level=warn ts=2026-04-28T13:13:52.638956996Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:55.22077654Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=warn ts=2026-04-28T13:13:57.639238697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:13:59.575185484Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:14:02.639686243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:14:03.808481507Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:14:03.930090985Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:14:06.202988006Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:14:07.639285036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:14:08.285551344Z caller=checkpoint.go:378 msg="writing series" size="319 kB"
level=info ts=2026-04-28T13:14:12.638772616Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:14:12.638797912Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:14:12.638790127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:14:12.638806394Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:14:12.63985194Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=7
level=info ts=2026-04-28T13:14:12.64576862Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000000.tmp new=/var/loki/wal/checkpoint.000000
level=info ts=2026-04-28T13:14:12.649073983Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009659914s
level=info ts=2026-04-28T13:14:13.904053162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:14:13.913410983Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:14:17.639639034Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:14:22.638704218Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:14:27.638967588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:14:32.639464386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:14:35.570760814Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:14:36.204751798Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:14:37.638649499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:14:42.622973757Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:14:42.623716346Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:14:42.638396458Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:14:42.638537885Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000001
level=warn ts=2026-04-28T13:14:42.638649028Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:14:42.638647708Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:14:42.638685548Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:14:42.638696595Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:14:42.638937168Z caller=checkpoint.go:378 msg="writing series" size="326 kB"
level=debug ts=2026-04-28T13:14:46.071024632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:14:46.846758483Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49904"
level=debug ts=2026-04-28T13:14:46.99436915Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:14:47.638847013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:14:51.34892949Z caller=checkpoint.go:378 msg="writing series" size="388 kB"
level=warn ts=2026-04-28T13:14:52.63863501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:14:55.703794914Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T13:14:56.732707671Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:14:56.743045321Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:14:57.639542489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:00.058477069Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=debug ts=2026-04-28T13:15:00.058499999Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-28T13:15:02.63884079Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:04.413361473Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:15:06.207488429Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:15:06.849142785Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:15:06.849205512Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:15:06.854798118Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:15:07.639557158Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:08.768029848Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-28T13:15:12.638924197Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:15:12.638949251Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:15:12.638958552Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:15:12.638964357Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:13.123708552Z caller=checkpoint.go:378 msg="writing series" size="104 kB"
level=debug ts=2026-04-28T13:15:17.477606748Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=warn ts=2026-04-28T13:15:17.638992399Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:21.833103406Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:15:22.63911467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:15:22.898473059Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:15:22.905028855Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:15:26.187724012Z caller=checkpoint.go:378 msg="writing series" size="48 kB"
level=warn ts=2026-04-28T13:15:27.638851741Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:30.542705029Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
level=warn ts=2026-04-28T13:15:32.63959941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:34.897897653Z caller=checkpoint.go:378 msg="writing series" size="29 kB"
ts=2026-04-28T13:15:36.209753841Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:15:36.913682414Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:15:37.639235913Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:39.252067592Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-28T13:15:42.623265043Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:15:42.623281747Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=debug ts=2026-04-28T13:15:42.623500488Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T13:15:42.623537819Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974868 n=1
level=debug ts=2026-04-28T13:15:42.623576259Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974868 n=1
level=info ts=2026-04-28T13:15:42.639517528Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:15:42.639534068Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:15:42.639541969Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:15:42.639554063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:43.607103146Z caller=checkpoint.go:378 msg="writing series" size="582 kB"
level=debug ts=2026-04-28T13:15:43.607126972Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=debug ts=2026-04-28T13:15:46.661456007Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:15:46.851267029Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:37866"
level=warn ts=2026-04-28T13:15:47.639426786Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:47.962341404Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T13:15:49.935034068Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:15:49.941455276Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:15:51.931998175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:15:51.93878568Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:15:52.316826597Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=warn ts=2026-04-28T13:15:52.639511419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:15:56.675365724Z caller=checkpoint.go:378 msg="writing series" size="5.4 MB"
level=debug ts=2026-04-28T13:15:56.675388329Z caller=checkpoint.go:390 msg="flushing series" totalSize="5.4 MB" series=3
level=warn ts=2026-04-28T13:15:57.639130978Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:01.026677008Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:16:02.638998138Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:04.053089312Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:16:05.381782565Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
ts=2026-04-28T13:16:06.212294931Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:16:07.638690024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:09.736061752Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-04-28T13:16:12.639017431Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:16:12.639042405Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:16:12.639050388Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:16:12.6390878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:14.090704115Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:16:17.639329862Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:18.445643778Z caller=checkpoint.go:378 msg="writing series" size="209 kB"
level=warn ts=2026-04-28T13:16:22.638833631Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:22.800122665Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:16:27.155735545Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:16:27.63880147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:31.433515847Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:16:31.433582002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:16:31.437348351Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:16:31.51052026Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=debug ts=2026-04-28T13:16:31.884152963Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:16:31.88630408Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:16:32.63870152Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:35.864814924Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:16:36.214808336Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:16:37.639027555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:40.220410488Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:16:41.619068493Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:16:41.623919666Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:16:42.382112903Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:16:42.387187718Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:16:42.622959397Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:16:42.623064837Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:16:42.639273094Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:16:42.639289232Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:16:42.639297119Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:16:42.63928831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:44.57540374Z caller=checkpoint.go:378 msg="writing series" size="277 kB"
ts=2026-04-28T13:16:46.856956542Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52514"
level=warn ts=2026-04-28T13:16:47.639130841Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:16:47.758445741Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:16:47.765200066Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:16:48.929311237Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-28T13:16:52.639036694Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:53.284519497Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-04-28T13:16:57.639312653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:16:57.639341844Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:17:01.993683494Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-28T13:17:01.99371005Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=15
level=warn ts=2026-04-28T13:17:02.639306008Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:05.477539958Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:17:06.216856754Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:17:06.348734542Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:17:07.639153604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:10.703367036Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-28T13:17:12.638760039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:17:12.63877548Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:17:12.63879854Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:17:12.63880907Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:17:15.059094173Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-28T13:17:16.8585636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41282"
level=warn ts=2026-04-28T13:17:17.639011244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:17:18.484247381Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:17:18.490838126Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:17:19.413438787Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-28T13:17:22.639080547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:23.768353243Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-28T13:17:24.391046162Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:17:24.396271607Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:17:27.182187622Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:17:27.638859733Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:28.123234088Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:17:32.479125335Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T13:17:32.479152246Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=7
level=warn ts=2026-04-28T13:17:32.639578094Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:17:36.219495102Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:17:36.83273651Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-28T13:17:37.305073817Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:17:37.313688774Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:17:37.639223517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:40.564373033Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:17:41.188212485Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=debug ts=2026-04-28T13:17:42.622491086Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:17:42.623536794Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:17:42.638898449Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:17:42.638919936Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:17:42.638906626Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:17:42.638927768Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:17:45.542706997Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T13:17:47.639622458Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:49.897135274Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-28T13:17:52.63944513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:54.251987971Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T13:17:57.639611966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:17:58.606797726Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:18:02.639464744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:02.961986253Z caller=checkpoint.go:378 msg="writing series" size="142 kB"
ts=2026-04-28T13:18:06.221739297Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=info ts=2026-04-28T13:18:06.522075998Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:18:06.529892672Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:18:07.317111696Z caller=checkpoint.go:378 msg="writing series" size="78 kB"
level=warn ts=2026-04-28T13:18:07.639285921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:18:11.194271468Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:18:11.198810027Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:18:11.671427475Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:18:12.63868346Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:18:12.638708798Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:18:12.638718163Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:18:12.638749885Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:13.336542721Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:18:16.026021076Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-28T13:18:17.638707656Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:20.38143023Z caller=checkpoint.go:378 msg="writing series" size="101 kB"
ts=2026-04-28T13:18:22.344157802Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60050"
level=warn ts=2026-04-28T13:18:22.638647821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:24.736267405Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:18:27.639558923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:27.947949377Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:18:29.09076062Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=debug ts=2026-04-28T13:18:29.090785844Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=13
level=info ts=2026-04-28T13:18:30.22506538Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:18:30.231602504Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:18:32.639158692Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:33.445878781Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:18:36.222608803Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:18:37.638845064Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:37.800239262Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T13:18:42.155442189Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-28T13:18:42.623241177Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:18:42.623767142Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:18:42.638695644Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:18:42.638714937Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:18:42.638724869Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:18:42.638731513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:44.77248612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:18:46.510167779Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T13:18:47.63914677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:18:49.194799879Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:18:49.200742313Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:18:50.865358304Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-28T13:18:52.638845244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:18:55.219827105Z caller=checkpoint.go:378 msg="writing series" size="308 kB"
level=warn ts=2026-04-28T13:18:57.638776414Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:18:57.889870658Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:18:57.895357845Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:18:59.574696224Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:19:02.639368923Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:19:03.929533295Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:19:06.224458269Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:19:07.639286494Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:19:08.284302324Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=debug ts=2026-04-28T13:19:08.284325429Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=9
level=debug ts=2026-04-28T13:19:12.165571489Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:19:12.639363828Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:19:12.639365819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:19:12.639390197Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:19:12.63939803Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:19:12.644679564Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000001.tmp new=/var/loki/wal/checkpoint.000001
level=info ts=2026-04-28T13:19:12.645870535Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007135447s
level=debug ts=2026-04-28T13:19:14.571465468Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T13:19:14.973654523Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:19:14.980918497Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:19:14.980938341Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:19:14.980961641Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:19:14.987470947Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:19:17.639456244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:19:22.346055326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37776"
level=warn ts=2026-04-28T13:19:22.639110386Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:19:27.639374563Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:19:31.085883494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:19:31.095432145Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:19:32.639654676Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:19:36.226876872Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:19:37.638886949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:19:42.622989478Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:19:42.623560515Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:19:42.639139345Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-28T13:19:42.639158029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:19:42.639176164Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:19:42.639192426Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:19:42.639199861Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:19:42.639287697Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000002
level=debug ts=2026-04-28T13:19:42.639693741Z caller=checkpoint.go:378 msg="writing series" size="403 kB"
level=debug ts=2026-04-28T13:19:42.681477024Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:19:43.049601526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:19:43.056137857Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:19:46.994461288Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:19:47.63867941Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:19:51.34934383Z caller=checkpoint.go:378 msg="writing series" size="327 kB"
level=warn ts=2026-04-28T13:19:52.639537839Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:19:52.930900975Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:19:52.936141782Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:19:55.7040253Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:19:57.639504416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:00.059485691Z caller=checkpoint.go:378 msg="writing series" size="239 kB"
level=warn ts=2026-04-28T13:20:02.638652194Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:04.413891471Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:20:06.229454086Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:20:06.837655826Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:20:07.639486501Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:08.768803797Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-28T13:20:08.768830546Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
level=debug ts=2026-04-28T13:20:09.835928092Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:20:10.800485914Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:20:10.80763219Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:20:12.638695841Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:20:12.638698604Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:12.638720903Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:20:12.638731438Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:20:13.12426506Z caller=checkpoint.go:378 msg="writing series" size="164 kB"
ts=2026-04-28T13:20:16.86737133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:45134"
level=debug ts=2026-04-28T13:20:17.478306913Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
level=warn ts=2026-04-28T13:20:17.639384176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:21.833586141Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:20:22.638905972Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:26.188727815Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-28T13:20:27.639343355Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:30.543732082Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=info ts=2026-04-28T13:20:32.532477687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:20:32.538799635Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:20:32.639619134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:34.898289482Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
ts=2026-04-28T13:20:36.231353674Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:20:37.639487625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:38.838329371Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:20:39.253423815Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T13:20:42.622978869Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:20:42.62343833Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:20:42.638957655Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:20:42.63898208Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:20:42.638990295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:20:42.638959036Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:43.607970271Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
ts=2026-04-28T13:20:44.963353145Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36696"
level=warn ts=2026-04-28T13:20:47.63944424Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:20:47.962700836Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-04-28T13:20:49.284124462Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:20:49.29157342Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:20:52.31741081Z caller=checkpoint.go:378 msg="writing series" size="62 kB"
ts=2026-04-28T13:20:52.350441954Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:56882"
level=warn ts=2026-04-28T13:20:52.639216378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:20:52.971695217Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:20:52.978448625Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:20:56.67821945Z caller=checkpoint.go:378 msg="writing series" size="6.1 MB"
level=debug ts=2026-04-28T13:20:56.678244015Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.2 MB" series=11
level=warn ts=2026-04-28T13:20:57.639266166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:01.027260111Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:21:02.638684576Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:03.023915845Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:21:03.025499349Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:21:05.38162482Z caller=checkpoint.go:378 msg="writing series" size="149 kB"
ts=2026-04-28T13:21:06.233164839Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:21:07.639409999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:08.876927211Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:21:09.736735639Z caller=checkpoint.go:378 msg="writing series" size="100 kB"
level=info ts=2026-04-28T13:21:12.639385033Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:21:12.639390808Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:12.63941099Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:21:12.639419837Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:21:14.091516886Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:21:17.63937382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:18.446421392Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
ts=2026-04-28T13:21:22.351768639Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50904"
level=warn ts=2026-04-28T13:21:22.639517517Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:22.801825465Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:21:27.1557022Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:21:27.639110264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:31.510726848Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=info ts=2026-04-28T13:21:32.106541844Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:21:32.113852572Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:21:32.113876107Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:21:32.113899386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:21:32.118544953Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:21:32.639215129Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:35.8655196Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:21:36.234744506Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:21:37.639130505Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:21:38.72970953Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:21:38.734118622Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:21:39.854414536Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:21:40.220413756Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:21:42.623067257Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:21:42.623104241Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:21:42.63932515Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:21:42.639345399Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:21:42.639354563Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:21:42.639397477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:44.575812923Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=info ts=2026-04-28T13:21:46.233433434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:21:46.238949962Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:21:47.63912905Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:21:48.515670116Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:21:48.522336713Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:21:48.930605525Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-28T13:21:52.639386443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:53.285422416Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-28T13:21:53.285449258Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-28T13:21:57.639141865Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:21:57.640192142Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:22:01.995391435Z caller=checkpoint.go:378 msg="writing series" size="5.7 kB"
level=warn ts=2026-04-28T13:22:02.639622341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:05.404489682Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:22:06.236259615Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:22:06.35012699Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:22:07.639335344Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:22:09.382684587Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:22:09.388124002Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:22:10.704253172Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=debug ts=2026-04-28T13:22:10.928119302Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:22:12.639039074Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:22:12.639034394Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:12.639060946Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:22:12.639078631Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:22:15.059355903Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:22:17.639165925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:19.414222805Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
ts=2026-04-28T13:22:22.354899207Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:59016"
level=warn ts=2026-04-28T13:22:22.638910716Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:23.769214983Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T13:22:27.638771039Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:28.124166059Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-04-28T13:22:30.729383757Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:22:30.734709906Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:22:32.479949874Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T13:22:32.479973212Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-04-28T13:22:32.639182263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:22:36.237219913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:22:36.833630188Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-04-28T13:22:37.638839551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:38.938080124Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:22:41.188522923Z caller=checkpoint.go:378 msg="writing series" size="230 kB"
level=info ts=2026-04-28T13:22:42.622979901Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:22:42.623835705Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:22:42.638754313Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:22:42.638758014Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:42.638775513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:22:42.638783753Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:22:44.341795384Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:22:44.349680952Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:22:44.975698475Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37920"
level=debug ts=2026-04-28T13:22:45.543700501Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=debug ts=2026-04-28T13:22:46.446131955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:22:47.638954134Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:49.897982972Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-28T13:22:52.639551802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:54.252429847Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:22:57.639618017Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:22:58.607725843Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-28T13:23:00.305714121Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:23:00.31176562Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:23:02.638861875Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:02.96234668Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
ts=2026-04-28T13:23:06.23859612Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:23:07.317300015Z caller=checkpoint.go:378 msg="writing series" size="84 kB"
level=warn ts=2026-04-28T13:23:07.639095279Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:10.665043746Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:23:11.671801517Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:23:12.639557635Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:23:12.639583702Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:23:12.639591836Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:23:12.639612835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:16.027676731Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
ts=2026-04-28T13:23:16.88200539Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:44868"
level=warn ts=2026-04-28T13:23:17.638941809Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:20.382236246Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T13:23:22.639124871Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:24.737216005Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:23:27.639398677Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:29.092052417Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=debug ts=2026-04-28T13:23:29.092076332Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=info ts=2026-04-28T13:23:30.342166986Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:23:30.348723077Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:23:30.348748004Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:23:30.348775589Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:23:30.352559322Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:23:32.63892562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:33.284451454Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:23:33.446959255Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:23:36.239877309Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:23:37.639085548Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:37.801532789Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-28T13:23:38.148925327Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:23:38.155976012Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:23:42.156455861Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-04-28T13:23:42.622649721Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:23:42.623686784Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:23:42.63930312Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:23:42.639322677Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:23:42.639330579Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:23:42.639347529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:46.511505608Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=debug ts=2026-04-28T13:23:46.588778582Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:23:47.216791371Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:23:47.223953135Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:23:47.638701443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:23:48.959136491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:23:48.9646371Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:23:50.865620425Z caller=checkpoint.go:378 msg="writing series" size="173 kB"
level=warn ts=2026-04-28T13:23:52.63940066Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:55.221232484Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
level=warn ts=2026-04-28T13:23:57.639626432Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:23:59.57575494Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:24:02.639289882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:03.930168723Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:24:06.24212433Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:24:07.639116043Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:08.285805904Z caller=checkpoint.go:378 msg="writing series" size="381 kB"
level=debug ts=2026-04-28T13:24:08.285832197Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=9
level=info ts=2026-04-28T13:24:12.638771463Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:24:12.638800085Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:24:12.63881207Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:24:12.638823426Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:24:12.645278554Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000002.tmp new=/var/loki/wal/checkpoint.000002
level=info ts=2026-04-28T13:24:12.646699438Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007269345s
level=info ts=2026-04-28T13:24:16.2114778Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:24:16.217911562Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:24:16.888011265Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:24:17.639114358Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:19.928774417Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:24:22.638645869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:24:24.303160854Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:24:24.311510274Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:24:27.638951328Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:24:32.639360595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:24:36.243705202Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:24:37.639492893Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:24:42.622939035Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:24:42.623471892Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:24:42.638444813Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:24:42.638611073Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-28T13:24:42.638621479Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:24:42.638642226Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:24:42.638650037Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:24:42.638651926Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:42.638937456Z caller=checkpoint.go:378 msg="writing series" size="265 kB"
ts=2026-04-28T13:24:44.987583706Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37464"
level=debug ts=2026-04-28T13:24:45.512565211Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:24:46.874518793Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:24:46.881325728Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:24:46.994712371Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:24:47.639284468Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:51.349287872Z caller=checkpoint.go:378 msg="writing series" size="461 kB"
level=debug ts=2026-04-28T13:24:51.515847052Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:24:51.51740643Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T13:24:52.363982071Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45000"
level=warn ts=2026-04-28T13:24:52.638646373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:24:55.703570648Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:24:57.639724165Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:00.0586219Z caller=checkpoint.go:378 msg="writing series" size="374 kB"
level=debug ts=2026-04-28T13:25:00.058644233Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=5
level=warn ts=2026-04-28T13:25:02.638631373Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:04.413282463Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:25:06.245328192Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:25:07.639520314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:08.768256389Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=info ts=2026-04-28T13:25:10.832844003Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:25:10.840291789Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:25:12.536915827Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:25:12.544525614Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:25:12.639500758Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:25:12.639525141Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:25:12.639532709Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:25:12.639499645Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:13.12346786Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-28T13:25:16.841725812Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:25:17.478474368Z caller=checkpoint.go:378 msg="writing series" size="248 kB"
level=warn ts=2026-04-28T13:25:17.639484191Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:21.833412349Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:25:22.639335514Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:26.187326697Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
level=info ts=2026-04-28T13:25:27.27144407Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:25:27.278614837Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:25:27.639249263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:30.543096515Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=warn ts=2026-04-28T13:25:32.639303784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:34.897419538Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-28T13:25:36.246343805Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:25:37.639695183Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:38.319751822Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:25:39.252396827Z caller=checkpoint.go:378 msg="writing series" size="2.2 kB"
level=debug ts=2026-04-28T13:25:42.622925314Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:25:42.622956414Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:25:42.639502665Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:25:42.639523467Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:42.639524352Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:25:42.639539749Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:25:43.343429356Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:25:43.607256675Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=debug ts=2026-04-28T13:25:43.607281089Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
level=warn ts=2026-04-28T13:25:47.639435717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:47.961690969Z caller=checkpoint.go:378 msg="writing series" size="4.1 kB"
level=info ts=2026-04-28T13:25:51.931261338Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:25:51.938432802Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:25:52.316582581Z caller=checkpoint.go:378 msg="writing series" size="71 kB"
level=warn ts=2026-04-28T13:25:52.639387639Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:25:56.675758379Z caller=checkpoint.go:378 msg="writing series" size="6.6 MB"
level=debug ts=2026-04-28T13:25:56.675782292Z caller=checkpoint.go:390 msg="flushing series" totalSize="6.6 MB" series=3
level=warn ts=2026-04-28T13:25:57.63915624Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:01.026317417Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:26:02.639603686Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:26:05.184062127Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:26:05.193284575Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:26:05.193306502Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:26:05.193330946Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:26:05.200192986Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:26:05.381820129Z caller=checkpoint.go:378 msg="writing series" size="165 kB"
ts=2026-04-28T13:26:06.247694634Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:26:07.63932773Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:08.557145458Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:26:09.736690932Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=info ts=2026-04-28T13:26:12.639333253Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:26:12.639339308Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:12.639357806Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:26:12.639365655Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:26:14.090885271Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-28T13:26:15.436717181Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:26:15.44436915Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:26:17.639107127Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:18.445420886Z caller=checkpoint.go:378 msg="writing series" size="44 kB"
level=info ts=2026-04-28T13:26:19.816277062Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:26:19.8240581Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:26:22.370391271Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:51154"
level=warn ts=2026-04-28T13:26:22.639196521Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:22.800548042Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:26:27.155835448Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:26:27.639736592Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:31.509972454Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:26:32.638925481Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:35.865147308Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:26:36.249252998Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:26:36.992030588Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:26:37.639252253Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:38.483632395Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:26:40.219725852Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-28T13:26:42.621982657Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:26:42.623045154Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:26:42.639004879Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:26:42.639019395Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:26:42.639027197Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:26:42.639040895Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:44.574833506Z caller=checkpoint.go:378 msg="writing series" size="304 kB"
ts=2026-04-28T13:26:45.001281952Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37246"
ts=2026-04-28T13:26:46.896949319Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41106"
level=warn ts=2026-04-28T13:26:47.638812269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:26:47.807898186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:26:47.813475005Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:26:48.725060724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:26:48.732523542Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:26:48.930131128Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-28T13:26:52.63921286Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:53.284914332Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=warn ts=2026-04-28T13:26:57.639515872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:26:57.639525512Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:27:01.994147131Z caller=checkpoint.go:378 msg="writing series" size="6.9 kB"
level=warn ts=2026-04-28T13:27:02.638941078Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:04.941527103Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:27:06.250305559Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:27:06.349213682Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-28T13:27:06.349240394Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
level=warn ts=2026-04-28T13:27:07.639461473Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:27:09.068719432Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:27:09.076255464Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:27:10.704057965Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:27:12.639199286Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:27:12.639220688Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:27:12.639229373Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:27:12.639195571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:27:15.005029804Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:47610"
level=debug ts=2026-04-28T13:27:15.058913136Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:27:17.638937107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:19.413982Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-28T13:27:22.638866726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:23.768005529Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T13:27:26.364794576Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:27:27.63904945Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:28.12275114Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-04-28T13:27:31.502719964Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:27:31.507322726Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:27:32.478899541Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-28T13:27:32.478935445Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=6
level=warn ts=2026-04-28T13:27:32.639338794Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:34.433436123Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:27:36.251551271Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:27:36.833110431Z caller=checkpoint.go:378 msg="writing series" size="93 kB"
level=warn ts=2026-04-28T13:27:37.639148625Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:27:40.444691922Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:27:40.452835584Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:27:41.187978155Z caller=checkpoint.go:378 msg="writing series" size="322 kB"
level=info ts=2026-04-28T13:27:42.622947855Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:27:42.622977761Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:27:42.639146925Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:27:42.639162881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:42.639177825Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:27:42.639188385Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:27:45.542770883Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
ts=2026-04-28T13:27:46.899355435Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:42870"
level=warn ts=2026-04-28T13:27:47.639123451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:49.897150037Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-28T13:27:52.638779965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:27:54.252729376Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-28T13:27:57.638969144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:27:58.42611438Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:27:58.432600061Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:27:58.43262152Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:27:58.432643284Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:27:58.437349887Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:27:58.606626213Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-28T13:28:01.909883718Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:28:02.63939776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:02.961838097Z caller=checkpoint.go:378 msg="writing series" size="156 kB"
ts=2026-04-28T13:28:06.253090532Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:28:07.316851275Z caller=checkpoint.go:378 msg="writing series" size="91 kB"
level=warn ts=2026-04-28T13:28:07.639561621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:08.796861335Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:28:11.671723578Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-28T13:28:12.638715606Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:28:12.638711458Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:28:12.638736127Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:28:12.638743651Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:28:15.012115049Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58540"
level=debug ts=2026-04-28T13:28:16.026693599Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-28T13:28:16.80577047Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:28:16.813767934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:28:16.900314193Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:35612"
level=warn ts=2026-04-28T13:28:17.63953844Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:20.381214933Z caller=checkpoint.go:378 msg="writing series" size="160 kB"
level=debug ts=2026-04-28T13:28:20.381238636Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-28T13:28:22.639474192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:24.735722703Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-28T13:28:25.867852924Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:28:25.874844273Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:28:27.112997201Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:28:27.117377645Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:28:27.638921993Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:29.090682874Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-04-28T13:28:32.638957037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:33.446199592Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-28T13:28:33.82355767Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:28:36.253887118Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:28:37.639600256Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:37.800972128Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-28T13:28:40.459094934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:28:40.466480297Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:28:42.155537621Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-04-28T13:28:42.622495187Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:28:42.623548609Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:28:42.638983688Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:28:42.639013235Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:28:42.639005642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:28:42.639021249Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:28:45.014885235Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:37344"
level=debug ts=2026-04-28T13:28:46.510476613Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
ts=2026-04-28T13:28:46.901379274Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:39064"
level=warn ts=2026-04-28T13:28:47.63959802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:50.865706195Z caller=checkpoint.go:378 msg="writing series" size="224 kB"
level=debug ts=2026-04-28T13:28:51.431915432Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:28:51.433716673Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
ts=2026-04-28T13:28:52.378374305Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40528"
level=warn ts=2026-04-28T13:28:52.639188266Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:55.220043427Z caller=checkpoint.go:378 msg="writing series" size="407 kB"
level=debug ts=2026-04-28T13:28:55.220065629Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-28T13:28:57.639750492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:28:59.575093912Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:29:02.638946949Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:02.747333835Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:29:03.929745714Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:29:06.256647929Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=info ts=2026-04-28T13:29:07.169462473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:29:07.176354688Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:29:07.639368821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:08.28509371Z caller=checkpoint.go:378 msg="writing series" size="396 kB"
level=info ts=2026-04-28T13:29:12.639369966Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:29:12.63937803Z caller=checkpoint.go:390 msg="flushing series" totalSize="546 kB" series=3
level=debug ts=2026-04-28T13:29:12.639399522Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:29:12.639409844Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:29:12.639405495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:29:12.645421033Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000003.tmp new=/var/loki/wal/checkpoint.000003
level=info ts=2026-04-28T13:29:12.64671356Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007946495s
level=info ts=2026-04-28T13:29:13.524516616Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:29:13.531571348Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:29:15.01767452Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:49494"
level=warn ts=2026-04-28T13:29:17.639242679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:29:22.379682816Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58520"
level=warn ts=2026-04-28T13:29:22.639520347Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:29:27.638634796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:29:32.639264748Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:32.763972991Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:29:34.15935506Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:29:35.728202984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:29:35.7332817Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:29:36.259885409Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:29:37.6395921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:42.622695882Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:29:42.62369657Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:29:42.638575606Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:29:42.638601872Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:29:42.638621932Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:29:42.638630661Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:29:42.638637623Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:29:42.638746589Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000004
level=debug ts=2026-04-28T13:29:42.639107372Z caller=checkpoint.go:378 msg="writing series" size="340 kB"
level=debug ts=2026-04-28T13:29:46.994565885Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:29:47.639435907Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:51.349111952Z caller=checkpoint.go:378 msg="writing series" size="601 kB"
level=warn ts=2026-04-28T13:29:52.638902644Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:29:55.704259751Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:29:57.639375524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:00.058524076Z caller=checkpoint.go:378 msg="writing series" size="273 kB"
level=debug ts=2026-04-28T13:30:00.058544581Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=info ts=2026-04-28T13:30:00.625085878Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:30:00.632675361Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:30:01.895188864Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:30:02.638785533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:30:03.248572608Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:30:03.253417637Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:30:03.253439755Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:30:03.253464497Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:30:03.25708199Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:30:04.413217615Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:30:06.262710807Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:30:07.639191711Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:08.768268608Z caller=checkpoint.go:378 msg="writing series" size="185 kB"
level=info ts=2026-04-28T13:30:12.638615732Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:30:12.638642488Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:30:12.638650801Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:30:12.638656088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:13.123534013Z caller=checkpoint.go:378 msg="writing series" size="292 kB"
level=info ts=2026-04-28T13:30:15.520929296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:30:15.527135829Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:30:17.478265673Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=warn ts=2026-04-28T13:30:17.638774696Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:18.991443632Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:30:21.83279495Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:30:22.639473083Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:30:23.066258926Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:30:23.071354191Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:30:26.187760239Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=warn ts=2026-04-28T13:30:27.639553192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:30.54272439Z caller=checkpoint.go:378 msg="writing series" size="45 kB"
level=warn ts=2026-04-28T13:30:32.639256059Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:34.897496081Z caller=checkpoint.go:378 msg="writing series" size="33 kB"
ts=2026-04-28T13:30:36.264551609Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:30:36.77660996Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:30:37.638702965Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:30:39.146994865Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:30:39.152376675Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:30:39.252745455Z caller=checkpoint.go:378 msg="writing series" size="2.5 kB"
level=info ts=2026-04-28T13:30:42.62310237Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:30:42.623177174Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=debug ts=2026-04-28T13:30:42.624442237Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T13:30:42.62448129Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974869 n=1
level=debug ts=2026-04-28T13:30:42.624523727Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974869 n=1
level=info ts=2026-04-28T13:30:42.638601404Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:30:42.638617836Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:30:42.6386261Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:30:42.638675327Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:43.607600032Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=debug ts=2026-04-28T13:30:43.607639291Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=10
ts=2026-04-28T13:30:46.910330707Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:50062"
level=warn ts=2026-04-28T13:30:47.639507052Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:30:47.962116335Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-28T13:30:52.316805409Z caller=checkpoint.go:378 msg="writing series" size="79 kB"
level=warn ts=2026-04-28T13:30:52.639574071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:30:52.798920434Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:30:52.808544942Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:30:56.678215285Z caller=checkpoint.go:378 msg="writing series" size="7.3 MB"
level=debug ts=2026-04-28T13:30:56.67824094Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.4 MB" series=3
level=warn ts=2026-04-28T13:30:57.639282868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:01.026565918Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=debug ts=2026-04-28T13:31:02.438406884Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:31:02.638870131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:31:05.20816526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:31:05.218612451Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:31:05.381984954Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
ts=2026-04-28T13:31:06.266792961Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:31:07.639207536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:09.736316214Z caller=checkpoint.go:378 msg="writing series" size="150 kB"
level=info ts=2026-04-28T13:31:12.639544426Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:31:12.639553492Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:12.63957026Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:31:12.639578681Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:31:14.090584045Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:31:17.638711433Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:18.445714946Z caller=checkpoint.go:378 msg="writing series" size="76 kB"
level=debug ts=2026-04-28T13:31:21.266923409Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:31:22.638823162Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:22.801212346Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:31:26.783830938Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:31:27.15590331Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:31:27.639138278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:31:29.60633471Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:31:29.61059555Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:31:31.510131407Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:31:32.639442826Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:35.865741226Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:31:36.269445113Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=info ts=2026-04-28T13:31:37.469481835Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:31:37.478743859Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:31:37.639484454Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:40.219631394Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:31:42.623140982Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:31:42.623144291Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:31:42.638653324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:31:42.638687315Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:31:42.638695663Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:31:42.638691615Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:44.575540336Z caller=checkpoint.go:378 msg="writing series" size="321 kB"
level=warn ts=2026-04-28T13:31:47.639505313Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:48.929373302Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-04-28T13:31:52.59580478Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:31:52.60261628Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:31:52.639139157Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:53.284501774Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-28T13:31:53.284526384Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.0 MB" series=13
level=debug ts=2026-04-28T13:31:57.639680394Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-28T13:31:57.639675337Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:31:58.077414048Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:32:01.994118993Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=warn ts=2026-04-28T13:32:02.639579887Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:32:06.271514861Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:32:06.349323629Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:32:07.639245527Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:09.184532138Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:32:10.704257741Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:32:11.631049828Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:32:11.634810168Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:32:12.63889405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:32:12.638888448Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:32:12.63892503Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:32:12.638935013Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:32:15.038435236Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59368"
level=debug ts=2026-04-28T13:32:15.058829245Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:32:17.639558366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:19.413879719Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
ts=2026-04-28T13:32:22.392571304Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41196"
level=warn ts=2026-04-28T13:32:22.639143228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:23.768476448Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T13:32:27.638791107Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:28.123484555Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-04-28T13:32:29.139508724Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:32:29.147844551Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:32:29.147869885Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:32:29.14790002Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:32:29.155032833Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:32:29.500944301Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:32:32.479676154Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T13:32:32.479699865Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-04-28T13:32:32.638849868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:32:36.274782673Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:32:36.832905946Z caller=checkpoint.go:378 msg="writing series" size="109 kB"
level=warn ts=2026-04-28T13:32:37.639617292Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:32:39.721524485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:32:39.730404098Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:32:41.188437777Z caller=checkpoint.go:378 msg="writing series" size="204 kB"
level=debug ts=2026-04-28T13:32:42.622318928Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:32:42.623371121Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:32:42.639010913Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:32:42.639017684Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:42.639029816Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:32:42.639039233Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:32:45.542496206Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=info ts=2026-04-28T13:32:46.277186972Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:32:46.283993872Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:32:46.918491429Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60584"
level=debug ts=2026-04-28T13:32:47.340501345Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:32:47.342231514Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:32:47.63897832Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:49.897074751Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-28T13:32:52.638790161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:54.25267602Z caller=checkpoint.go:378 msg="writing series" size="16 kB"
level=debug ts=2026-04-28T13:32:55.841288505Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:32:57.639398789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:32:58.607487606Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:33:02.639487109Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:02.961942692Z caller=checkpoint.go:378 msg="writing series" size="183 kB"
level=info ts=2026-04-28T13:33:03.824284684Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:33:03.829645172Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:33:06.277222832Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:33:07.316701284Z caller=checkpoint.go:378 msg="writing series" size="97 kB"
level=warn ts=2026-04-28T13:33:07.63863921Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:33:09.651689697Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:33:09.658854958Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:33:11.671342621Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:33:12.638694302Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:33:12.63872325Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:33:12.638730723Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:33:12.638725546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:33:15.044676897Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:44584"
level=debug ts=2026-04-28T13:33:16.026846822Z caller=checkpoint.go:378 msg="writing series" size="19 kB"
ts=2026-04-28T13:33:16.919854636Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:33950"
level=warn ts=2026-04-28T13:33:17.639630076Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:20.381772038Z caller=checkpoint.go:378 msg="writing series" size="192 kB"
level=debug ts=2026-04-28T13:33:20.38179802Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=info ts=2026-04-28T13:33:21.767714484Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:33:21.777632645Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:33:22.639313487Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:24.736502728Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-28T13:33:26.018803955Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:33:27.639562035Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:29.091287994Z caller=checkpoint.go:378 msg="writing series" size="145 kB"
level=debug ts=2026-04-28T13:33:30.797016667Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:33:32.639544166Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:33.446287401Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:33:36.279439356Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:33:37.639453022Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:37.800848526Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T13:33:42.155305369Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-04-28T13:33:42.623239175Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:33:42.62377954Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:33:42.638817003Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:33:42.638841604Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:33:42.638849612Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:33:42.638863815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:33:45.047980008Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53520"
level=debug ts=2026-04-28T13:33:46.510284587Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-04-28T13:33:46.901708145Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:33:46.905893044Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:33:47.638907932Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:50.865655708Z caller=checkpoint.go:378 msg="writing series" size="74 kB"
ts=2026-04-28T13:33:52.398629877Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:48300"
level=warn ts=2026-04-28T13:33:52.638713835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:55.220594432Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=debug ts=2026-04-28T13:33:56.067768046Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:33:56.072843877Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=2 total_comp="4.7 MB" avg_comp="2.3 MB" total_uncomp="41 MB" avg_uncomp="20 MB" full=1 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:33:56.081163701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 2, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:33:57.639437849Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:33:59.57484017Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-28T13:33:59.574865673Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=9
level=warn ts=2026-04-28T13:34:02.639533389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:34:03.92989893Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T13:34:04.030999386Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:34:06.281355504Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:34:07.639560081Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:34:08.284685975Z caller=checkpoint.go:378 msg="writing series" size="431 kB"
level=info ts=2026-04-28T13:34:12.638680763Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:34:12.638711088Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:34:12.63871263Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:34:12.63872381Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:34:12.639796951Z caller=checkpoint.go:390 msg="flushing series" totalSize="484 kB" series=2
level=info ts=2026-04-28T13:34:12.645632451Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000004.tmp new=/var/loki/wal/checkpoint.000004
level=info ts=2026-04-28T13:34:12.647045022Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.00815971s
level=warn ts=2026-04-28T13:34:17.63885929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:34:19.534076974Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:34:19.540686943Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:34:19.540710941Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:34:19.540736448Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:34:19.545410737Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:34:22.639198931Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:34:27.639046051Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:34:28.705167538Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:34:32.638642848Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:34:33.389989541Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:34:33.395982623Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:34:34.393861485Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:34:34.399087245Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:34:36.283148051Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:34:37.639210869Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:34:42.62251266Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:34:42.623560792Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:34:42.638595362Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=info ts=2026-04-28T13:34:42.63860738Z caller=checkpoint.go:612 msg="starting checkpoint"
level=debug ts=2026-04-28T13:34:42.63861961Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:34:42.638627717Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:34:42.638649068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:34:42.638768924Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000005
level=debug ts=2026-04-28T13:34:42.639173323Z caller=checkpoint.go:378 msg="writing series" size="427 kB"
level=debug ts=2026-04-28T13:34:45.855318036Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:34:46.925947941Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:53898"
level=debug ts=2026-04-28T13:34:46.994370227Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:34:47.639391531Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:34:50.992580572Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:34:50.999093578Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:34:51.34938149Z caller=checkpoint.go:378 msg="writing series" size="474 kB"
level=warn ts=2026-04-28T13:34:52.638707966Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:34:55.70404934Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=info ts=2026-04-28T13:34:56.996696414Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:34:57.001964338Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:34:57.638917992Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:00.059353699Z caller=checkpoint.go:378 msg="writing series" size="397 kB"
level=debug ts=2026-04-28T13:35:00.05937985Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=5
level=warn ts=2026-04-28T13:35:02.639491024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:04.314746738Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:35:04.414000749Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:35:06.285498766Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:35:07.639168918Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:08.768781489Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=info ts=2026-04-28T13:35:12.639226063Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:35:12.639234522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:12.63925414Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:35:12.639263598Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:35:13.12385741Z caller=checkpoint.go:378 msg="writing series" size="138 kB"
ts=2026-04-28T13:35:16.926845712Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54030"
level=info ts=2026-04-28T13:35:17.054709285Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:35:17.059642428Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:35:17.478776187Z caller=checkpoint.go:378 msg="writing series" size="123 kB"
level=warn ts=2026-04-28T13:35:17.639314789Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:21.833079044Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T13:35:22.639182178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:26.187834856Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
level=warn ts=2026-04-28T13:35:27.639160244Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:30.542627124Z caller=checkpoint.go:378 msg="writing series" size="51 kB"
level=warn ts=2026-04-28T13:35:32.638657247Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:34.897770125Z caller=checkpoint.go:378 msg="writing series" size="34 kB"
ts=2026-04-28T13:35:36.288845239Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=info ts=2026-04-28T13:35:36.899901294Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:35:36.906211035Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:35:37.444752199Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:35:37.639629242Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:38.329359498Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:35:39.252042731Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
level=debug ts=2026-04-28T13:35:42.622112845Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:35:42.623155361Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:35:42.638830555Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:35:42.638852709Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:35:42.638860408Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:35:42.638873072Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:43.607709016Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=debug ts=2026-04-28T13:35:43.607732218Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=10
level=info ts=2026-04-28T13:35:44.322707579Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:35:44.328177667Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:35:45.060426563Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58520"
level=warn ts=2026-04-28T13:35:47.638778572Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:47.961965907Z caller=checkpoint.go:378 msg="writing series" size="4.8 kB"
level=debug ts=2026-04-28T13:35:52.317516581Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
level=warn ts=2026-04-28T13:35:52.63942551Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:35:56.677494668Z caller=checkpoint.go:378 msg="writing series" size="7.6 MB"
level=debug ts=2026-04-28T13:35:56.677518336Z caller=checkpoint.go:390 msg="flushing series" totalSize="7.7 MB" series=3
level=warn ts=2026-04-28T13:35:57.638920087Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:01.026644979Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:36:02.638644361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:02.935140894Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:36:05.38104606Z caller=checkpoint.go:378 msg="writing series" size="210 kB"
ts=2026-04-28T13:36:06.290949395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:36:07.638929536Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:09.736436122Z caller=checkpoint.go:378 msg="writing series" size="176 kB"
level=info ts=2026-04-28T13:36:12.137034425Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:36:12.144120157Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:36:12.144142738Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:36:12.144166317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:36:12.148215283Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:36:12.638683526Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:36:12.638709822Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:36:12.63871854Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:36:12.638719708Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:14.091447003Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:36:17.639156545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:18.446429735Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=warn ts=2026-04-28T13:36:22.639712815Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:22.801062742Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-28T13:36:25.501886322Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:36:25.511647105Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:36:27.15534835Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:36:27.639410845Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:36:27.873796861Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:36:27.878999516Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:36:29.147123469Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:36:29.153385479Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:36:29.773621664Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:36:29.775224495Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:36:31.510167894Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:36:32.639111139Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:35.865583378Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:36:36.291937109Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:36:37.21319291Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:36:37.639147882Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:40.220439083Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:36:42.623042298Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:36:42.623552246Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:36:42.63912522Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:36:42.639156907Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:36:42.63916486Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:36:42.639167489Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:44.575582004Z caller=checkpoint.go:378 msg="writing series" size="331 kB"
ts=2026-04-28T13:36:45.06673527Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34788"
level=warn ts=2026-04-28T13:36:47.639345117Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:48.930093702Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=info ts=2026-04-28T13:36:50.296955999Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:36:50.301781406Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:36:52.639493163Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:53.284722365Z caller=checkpoint.go:378 msg="writing series" size="125 kB"
level=debug ts=2026-04-28T13:36:53.284751366Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=13
level=warn ts=2026-04-28T13:36:57.638964048Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:36:57.63897664Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:37:01.994712484Z caller=checkpoint.go:378 msg="writing series" size="9.0 kB"
level=warn ts=2026-04-28T13:37:02.638954345Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:37:06.293908626Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:37:06.349577337Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-28T13:37:06.933987032Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:37:07.639424187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:37:08.229702014Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:37:08.237195407Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:37:10.703827512Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=warn ts=2026-04-28T13:37:12.639325867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:37:12.639325045Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:37:12.639352328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:37:12.63935974Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:37:15.059125706Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-28T13:37:16.935124622Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48868"
level=warn ts=2026-04-28T13:37:17.639339546Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:19.413791641Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-28T13:37:20.038709648Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:37:20.04774833Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:37:22.639646333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:23.768453547Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T13:37:27.639720401Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:28.123514744Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:37:30.290451419Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:37:32.479232617Z caller=checkpoint.go:378 msg="writing series" size="1.3 MB"
level=debug ts=2026-04-28T13:37:32.479254451Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=9
level=warn ts=2026-04-28T13:37:32.638835846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:36.066265361Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:37:36.296216136Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:37:36.832548226Z caller=checkpoint.go:378 msg="writing series" size="117 kB"
level=warn ts=2026-04-28T13:37:37.639334085Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:37:40.098647795Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:37:40.104605669Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:37:41.187958908Z caller=checkpoint.go:378 msg="writing series" size="332 kB"
level=debug ts=2026-04-28T13:37:42.622829849Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:37:42.623889106Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:37:42.639124139Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:37:42.639142336Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:37:42.639149922Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:37:42.639149962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:37:45.073391861Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56848"
level=debug ts=2026-04-28T13:37:45.543039841Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T13:37:47.639131785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:49.897411994Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=warn ts=2026-04-28T13:37:52.639264868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:54.252880341Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:37:57.639608974Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:37:58.606735499Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:38:02.638650737Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:02.96219384Z caller=checkpoint.go:378 msg="writing series" size="170 kB"
level=info ts=2026-04-28T13:38:04.876574539Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:38:04.88314337Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:38:06.298743011Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:38:07.317310354Z caller=checkpoint.go:378 msg="writing series" size="103 kB"
level=warn ts=2026-04-28T13:38:07.638898068Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:11.063123132Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:38:11.671860464Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:38:12.638647324Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:38:12.638699222Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:38:12.638695798Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:38:12.638708353Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:38:12.645900184Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:38:12.6523465Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:38:15.076747172Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54966"
level=debug ts=2026-04-28T13:38:16.026684176Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-28T13:38:17.639552061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:19.949294755Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:38:20.381522599Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-28T13:38:20.381550611Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=11
level=warn ts=2026-04-28T13:38:22.639641445Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:24.73637915Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:38:27.638908744Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:29.09130493Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T13:38:32.63957925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:33.446453241Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-28T13:38:35.492149676Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:38:35.49774514Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:38:36.301239808Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:38:37.638689717Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:37.80111103Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T13:38:42.155593782Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=info ts=2026-04-28T13:38:42.622949159Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:38:42.622960178Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:38:42.639385391Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:38:42.639404387Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:38:42.63941387Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:38:42.639420937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:43.236348703Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:38:45.07980906Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:55122"
level=debug ts=2026-04-28T13:38:46.510178139Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
ts=2026-04-28T13:38:46.94202664Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:57054"
level=warn ts=2026-04-28T13:38:47.639074463Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:38:48.668465091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:38:48.674512831Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:38:50.865261817Z caller=checkpoint.go:378 msg="writing series" size="127 kB"
level=warn ts=2026-04-28T13:38:52.638877098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:38:55.220024707Z caller=checkpoint.go:378 msg="writing series" size="468 kB"
level=debug ts=2026-04-28T13:38:55.22004995Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-28T13:38:57.639141323Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:38:58.466839439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:38:58.47456777Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:38:58.474589873Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:38:58.474614988Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:38:58.481644345Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:38:59.574984346Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:39:02.639290243Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:03.929741538Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:39:06.302118339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:39:07.639263881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:08.284393254Z caller=checkpoint.go:378 msg="writing series" size="497 kB"
level=debug ts=2026-04-28T13:39:10.822811639Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T13:39:12.638622487Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:39:12.63865178Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:39:12.638675966Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:39:12.638656086Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:12.639635362Z caller=checkpoint.go:390 msg="flushing series" totalSize="647 kB" series=3
level=info ts=2026-04-28T13:39:12.645410446Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000005.tmp new=/var/loki/wal/checkpoint.000005
level=info ts=2026-04-28T13:39:12.646882828Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007971815s
level=info ts=2026-04-28T13:39:12.710823717Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:39:12.721372186Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:39:16.942991677Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:52862"
level=warn ts=2026-04-28T13:39:17.639131755Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:17.907431901Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:39:22.419911901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:37072"
level=warn ts=2026-04-28T13:39:22.639505581Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:39:27.639640891Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:39:32.639321474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:39:32.789704089Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:39:32.795831668Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:39:36.30491707Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:39:37.639181461Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:39:41.259853311Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:39:41.26499038Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:39:42.622402701Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:39:42.623466044Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:39:42.639161186Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:39:42.639185891Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:39:42.639197433Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:39:42.639187772Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-28T13:39:42.639221322Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:39:42.639335159Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000006
level=debug ts=2026-04-28T13:39:42.639611351Z caller=checkpoint.go:378 msg="writing series" size="287 kB"
ts=2026-04-28T13:39:45.086622437Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56770"
level=info ts=2026-04-28T13:39:46.509588613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:39:46.516862126Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:39:46.995338682Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:39:47.639474264Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:48.732071181Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:39:51.350329882Z caller=checkpoint.go:378 msg="writing series" size="614 kB"
level=warn ts=2026-04-28T13:39:52.639491943Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:39:55.705014227Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:39:57.639629258Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:00.059879736Z caller=checkpoint.go:378 msg="writing series" size="293 kB"
level=debug ts=2026-04-28T13:40:00.05990561Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=5
level=warn ts=2026-04-28T13:40:02.638647804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:04.414003857Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:40:06.306862388Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:40:07.63883588Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:08.768782648Z caller=checkpoint.go:378 msg="writing series" size="186 kB"
level=debug ts=2026-04-28T13:40:09.775574447Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T13:40:12.639354163Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:40:12.639377217Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:40:12.639387291Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:40:12.639439282Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:13.001137068Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:40:13.123867891Z caller=checkpoint.go:378 msg="writing series" size="202 kB"
ts=2026-04-28T13:40:16.948012925Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46044"
level=debug ts=2026-04-28T13:40:17.478768374Z caller=checkpoint.go:378 msg="writing series" size="177 kB"
level=warn ts=2026-04-28T13:40:17.639080795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:40:19.466853125Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:40:19.472276989Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:40:21.833111553Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-28T13:40:22.424574115Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54992"
level=warn ts=2026-04-28T13:40:22.639469271Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:26.188519307Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=warn ts=2026-04-28T13:40:27.639448499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:30.543451379Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=warn ts=2026-04-28T13:40:32.639472342Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:34.897715477Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
ts=2026-04-28T13:40:36.309317683Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:40:37.639345009Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:40:38.082888655Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:40:38.091240436Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:40:38.637994186Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:40:38.643508823Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:40:38.643529836Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:40:38.643555136Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:40:38.648175149Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:40:39.253393663Z caller=checkpoint.go:378 msg="writing series" size="3.0 kB"
level=debug ts=2026-04-28T13:40:42.622542043Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:40:42.623605441Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:40:42.638617398Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:40:42.638649204Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:40:42.63864821Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:40:42.63865701Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:40:43.237508491Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:40:43.607893041Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
level=debug ts=2026-04-28T13:40:43.607916804Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=10
ts=2026-04-28T13:40:45.092862961Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57166"
level=debug ts=2026-04-28T13:40:46.826542932Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:40:46.828430269Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:40:47.639350464Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:47.96228851Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=info ts=2026-04-28T13:40:50.52693283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:40:50.536048016Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:40:52.317573888Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:40:52.639392178Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:40:56.677828729Z caller=checkpoint.go:378 msg="writing series" size="8.2 MB"
level=debug ts=2026-04-28T13:40:56.67785256Z caller=checkpoint.go:390 msg="flushing series" totalSize="8.3 MB" series=3
level=warn ts=2026-04-28T13:40:57.639405318Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:01.026867203Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:41:02.638861817Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:05.381811635Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
ts=2026-04-28T13:41:06.312574022Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:41:07.639464683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:09.736854869Z caller=checkpoint.go:378 msg="writing series" size="198 kB"
level=warn ts=2026-04-28T13:41:12.639144336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:41:12.639152138Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:41:12.639172775Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:41:12.639180555Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:41:13.537104283Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:41:13.543932256Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:41:14.091202034Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=debug ts=2026-04-28T13:41:15.32582329Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:41:17.639053054Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:18.446805873Z caller=checkpoint.go:378 msg="writing series" size="139 kB"
level=warn ts=2026-04-28T13:41:22.639508042Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:22.801805535Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-28T13:41:23.850966938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:41:23.861528626Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:41:25.208100494Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:41:25.213889437Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:41:27.156286262Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:41:27.638926872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:31.510747844Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:41:32.638675114Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:35.865476812Z caller=checkpoint.go:378 msg="writing series" size="2.6 kB"
ts=2026-04-28T13:41:36.314426466Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:41:37.63967567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:38.031327123Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:41:40.221105188Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-28T13:41:40.307309016Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:41:42.622463466Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:41:42.623516632Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:41:42.639451127Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:41:42.639473192Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:41:42.639480828Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:41:42.639475924Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:44.575911165Z caller=checkpoint.go:378 msg="writing series" size="346 kB"
ts=2026-04-28T13:41:46.955568561Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:59906"
level=warn ts=2026-04-28T13:41:47.639193611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:48.930681195Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=debug ts=2026-04-28T13:41:48.930707484Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=12
level=warn ts=2026-04-28T13:41:52.639598823Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:53.284965402Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=info ts=2026-04-28T13:41:56.983542225Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:41:56.991752759Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:41:57.638883925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:41:57.640006574Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:42:01.994427397Z caller=checkpoint.go:378 msg="writing series" size="9.8 kB"
level=warn ts=2026-04-28T13:42:02.63928834Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:42:06.315459678Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:42:06.349192496Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:42:07.639586254Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:42:08.486479307Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:42:08.4947044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:42:10.252917445Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:42:10.260598877Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:42:10.704896785Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:42:12.639178474Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:42:12.639187961Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:12.639204598Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:42:12.639212517Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:42:15.059444772Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
ts=2026-04-28T13:42:15.103367365Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54112"
level=info ts=2026-04-28T13:42:15.38172938Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="478 kB" avg_comp="478 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:42:15.38174846Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=error ts=2026-04-28T13:42:15.387687831Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:42:17.63917578Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:19.414534427Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=warn ts=2026-04-28T13:42:22.638985056Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:23.769012232Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T13:42:27.639500641Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:42:27.64464106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:42:27.651210368Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:42:28.123739997Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:42:32.479887742Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-28T13:42:32.479911452Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.7 MB" series=10
level=warn ts=2026-04-28T13:42:32.63948758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:42:36.316685222Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:42:36.833909774Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=warn ts=2026-04-28T13:42:37.638836425Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:37.852306551Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:42:40.37586517Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:42:41.188338405Z caller=checkpoint.go:378 msg="writing series" size="455 kB"
level=debug ts=2026-04-28T13:42:42.622408505Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:42:42.623458003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:42:42.638959799Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:42:42.638975088Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:42:42.638983242Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:42:42.639006125Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:42:45.106268694Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:34128"
level=debug ts=2026-04-28T13:42:45.543121546Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T13:42:47.639272161Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:42:49.485119087Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:42:49.490970172Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:42:49.490993294Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:42:49.491015241Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:42:49.494567173Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:42:49.897885999Z caller=checkpoint.go:378 msg="writing series" size="514 B"
level=info ts=2026-04-28T13:42:51.371327349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:42:51.378929231Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:42:52.532224022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:42:52.537309649Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:42:52.639233671Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:54.252705065Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-28T13:42:56.928293175Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:42:56.932836951Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:42:57.638646726Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:42:58.608040469Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:43:02.639332621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:02.963088826Z caller=checkpoint.go:378 msg="writing series" size="197 kB"
level=debug ts=2026-04-28T13:43:02.963105264Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=info ts=2026-04-28T13:43:03.93095028Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:43:03.937420119Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:43:06.318979566Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:43:07.317181565Z caller=checkpoint.go:378 msg="writing series" size="110 kB"
level=warn ts=2026-04-28T13:43:07.639653346Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:11.671818665Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=warn ts=2026-04-28T13:43:12.63876506Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:43:12.638792975Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:43:12.638815879Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:43:12.638825959Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:43:13.840593023Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:43:16.027694483Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=warn ts=2026-04-28T13:43:17.639542785Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:20.382597777Z caller=checkpoint.go:378 msg="writing series" size="253 kB"
level=warn ts=2026-04-28T13:43:22.638718713Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:43:24.705611187Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:43:24.70920826Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:43:24.736486135Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-28T13:43:26.283168424Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:43:27.639274451Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:29.09196382Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T13:43:32.639217533Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:33.446409318Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-28T13:43:34.802426349Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:43:34.809371243Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:43:36.321133878Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=info ts=2026-04-28T13:43:37.523118992Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:43:37.527699605Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:43:37.639534077Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:37.801900028Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T13:43:42.15648123Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-04-28T13:43:42.62228532Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:43:42.623359895Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:43:42.638681998Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:43:42.638710103Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:43:42.638719679Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:43:42.638717769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:43.980526635Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:43:46.510743889Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T13:43:47.639413507Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:43:49.561605537Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:43:49.57021276Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:43:50.865711053Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=debug ts=2026-04-28T13:43:50.865735961Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-28T13:43:52.638777172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:55.220649954Z caller=checkpoint.go:378 msg="writing series" size="348 kB"
level=warn ts=2026-04-28T13:43:57.639556683Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:43:59.575300773Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:44:02.639177827Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:03.929893141Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:44:06.322360446Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:44:07.639310304Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:08.28606704Z caller=checkpoint.go:378 msg="writing series" size="501 kB"
level=info ts=2026-04-28T13:44:12.638647205Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:44:12.638653477Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:12.638709241Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:44:12.638723126Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:44:12.639684362Z caller=checkpoint.go:390 msg="flushing series" totalSize="998 kB" series=4
level=info ts=2026-04-28T13:44:12.645924836Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000006.tmp new=/var/loki/wal/checkpoint.000006
level=info ts=2026-04-28T13:44:12.647330153Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.007872817s
level=warn ts=2026-04-28T13:44:17.639637668Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:44:18.634990148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:44:18.64199293Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:44:19.4056851Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:44:21.48614267Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:44:21.490129672Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:44:22.442731741Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:41748"
level=warn ts=2026-04-28T13:44:22.639437649Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:23.258025799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:44:27.638912562Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:44:29.860513721Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:44:29.867959758Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:44:31.93541545Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:44:31.939602684Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:44:32.639421124Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:44:36.323942378Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:44:37.639134805Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:44:42.623167588Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:44:42.623196823Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:44:42.639110891Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-28T13:44:42.639117693Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:44:42.639122792Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:44:42.63913915Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:44:42.639146617Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:44:42.639235044Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000007
level=debug ts=2026-04-28T13:44:42.639551479Z caller=checkpoint.go:378 msg="writing series" size="370 kB"
level=debug ts=2026-04-28T13:44:46.994969232Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:44:47.639084646Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:51.350138498Z caller=checkpoint.go:378 msg="writing series" size="737 kB"
level=debug ts=2026-04-28T13:44:51.350162908Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-04-28T13:44:52.444211485Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:50296"
level=warn ts=2026-04-28T13:44:52.638720779Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:52.86850579Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:44:55.704326264Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:44:57.639373429Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:44:58.139498635Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:44:58.141292162Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:45:00.059743481Z caller=checkpoint.go:378 msg="writing series" size="423 kB"
level=warn ts=2026-04-28T13:45:02.639393979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:04.413958324Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:45:06.326034072Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:45:07.638783727Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:08.768801816Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=info ts=2026-04-28T13:45:12.639491786Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:45:12.63950215Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:12.6395175Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:45:12.639531821Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:45:13.12411157Z caller=checkpoint.go:378 msg="writing series" size="270 kB"
ts=2026-04-28T13:45:15.12281067Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:45750"
level=info ts=2026-04-28T13:45:16.739422966Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:45:16.74613963Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:45:16.968605716Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58144"
level=debug ts=2026-04-28T13:45:17.478797591Z caller=checkpoint.go:378 msg="writing series" size="234 kB"
level=debug ts=2026-04-28T13:45:17.478823148Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-28T13:45:17.638842144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:45:18.55681095Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:45:18.562330752Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:45:18.875401612Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:45:19.492932803Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:45:19.497200338Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:45:21.833021889Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-28T13:45:22.445823766Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60296"
level=warn ts=2026-04-28T13:45:22.639591795Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:45:26.184277866Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:45:26.188553234Z caller=checkpoint.go:378 msg="writing series" size="159 kB"
level=error ts=2026-04-28T13:45:26.192595234Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:45:26.192614012Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:45:26.192636464Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:45:26.199516752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:45:27.63900413Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:30.543166264Z caller=checkpoint.go:378 msg="writing series" size="60 kB"
level=warn ts=2026-04-28T13:45:32.639467894Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:33.868701293Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:45:34.898371158Z caller=checkpoint.go:378 msg="writing series" size="37 kB"
ts=2026-04-28T13:45:36.328514149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:45:37.63936252Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:39.252640152Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-28T13:45:42.622279939Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:45:42.623329539Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:45:42.623513508Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T13:45:42.623561311Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974870 n=1
level=debug ts=2026-04-28T13:45:42.623602912Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974870 n=1
level=info ts=2026-04-28T13:45:42.638680748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:45:42.638702513Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:45:42.638710484Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:45:42.638706814Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:43.607983506Z caller=checkpoint.go:378 msg="writing series" size="483 kB"
ts=2026-04-28T13:45:45.125972524Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:33934"
level=info ts=2026-04-28T13:45:45.683868907Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:45:45.691008936Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:45:45.882693164Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:45:47.639412595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:47.963019488Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=debug ts=2026-04-28T13:45:52.317786673Z caller=checkpoint.go:378 msg="writing series" size="105 kB"
ts=2026-04-28T13:45:52.447034326Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54648"
level=warn ts=2026-04-28T13:45:52.639331674Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:45:56.680627468Z caller=checkpoint.go:378 msg="writing series" size="9.1 MB"
level=debug ts=2026-04-28T13:45:56.680651051Z caller=checkpoint.go:390 msg="flushing series" totalSize="9.9 MB" series=9
level=warn ts=2026-04-28T13:45:57.638973446Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:01.026862921Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=info ts=2026-04-28T13:46:02.015695399Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:46:02.022288979Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:46:02.639268421Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:46:05.135037613Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:46:05.13959861Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:46:05.382262905Z caller=checkpoint.go:378 msg="writing series" size="264 kB"
ts=2026-04-28T13:46:06.330564527Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=info ts=2026-04-28T13:46:06.987784157Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:46:06.994791828Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:46:07.639394226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:09.736359873Z caller=checkpoint.go:378 msg="writing series" size="222 kB"
level=info ts=2026-04-28T13:46:12.110908522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:46:12.115252971Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:46:12.639208493Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:46:12.639231799Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:46:12.639239943Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:46:12.63923466Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:14.091075297Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=warn ts=2026-04-28T13:46:17.63965393Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:18.446475764Z caller=checkpoint.go:378 msg="writing series" size="169 kB"
level=debug ts=2026-04-28T13:46:20.555816811Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:46:22.448411918Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:43562"
level=warn ts=2026-04-28T13:46:22.638847228Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:22.80126385Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:46:27.156384784Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:46:27.638967188Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:30.793327781Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:46:31.511443493Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:46:32.638866349Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:35.866132718Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
ts=2026-04-28T13:46:36.332620739Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:46:37.63939197Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:40.22009033Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:46:42.623245113Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:46:42.623839655Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:46:42.639580589Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:46:42.639596361Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:46:42.639603849Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:46:42.639616278Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:44.575832905Z caller=checkpoint.go:378 msg="writing series" size="357 kB"
level=debug ts=2026-04-28T13:46:44.575861299Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=11
level=warn ts=2026-04-28T13:46:47.638871999Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:48.930514391Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=debug ts=2026-04-28T13:46:51.792314058Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:46:52.449584749Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:44376"
level=warn ts=2026-04-28T13:46:52.639065495Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:53.284969651Z caller=checkpoint.go:378 msg="writing series" size="126 kB"
level=info ts=2026-04-28T13:46:54.007850428Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:46:54.017978618Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:46:55.049813118Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:46:55.054389568Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=info ts=2026-04-28T13:46:55.994901318Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:46:55.998975626Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:46:57.63913518Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:46:57.640245408Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-28T13:47:01.153779359Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:47:01.160882668Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:47:01.160901253Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:47:01.160921025Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:47:01.165811086Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:47:01.995121553Z caller=checkpoint.go:378 msg="writing series" size="11 kB"
level=warn ts=2026-04-28T13:47:02.639637167Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:47:06.333928144Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:47:06.349701543Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=warn ts=2026-04-28T13:47:07.639479419Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:10.704674245Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:47:12.639458005Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:47:12.639456325Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:12.639480711Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:47:12.639489019Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:47:15.058914865Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:47:17.638943236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:19.413943295Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-28T13:47:20.403193843Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:47:20.408333044Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:47:20.680653701Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:47:22.639261236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:23.768783061Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=debug ts=2026-04-28T13:47:25.775354678Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:47:27.638971058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:28.124045134Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:47:32.480114406Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-28T13:47:32.4801364Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.9 MB" series=11
level=warn ts=2026-04-28T13:47:32.639454534Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:47:36.334944641Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:47:36.833211415Z caller=checkpoint.go:378 msg="writing series" size="153 kB"
level=warn ts=2026-04-28T13:47:37.639245863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:41.188181683Z caller=checkpoint.go:378 msg="writing series" size="344 kB"
level=info ts=2026-04-28T13:47:42.623242603Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:47:42.624107315Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:47:42.639561509Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:47:42.639581469Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:47:42.639571225Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:47:42.63958892Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:47:44.36550629Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:47:44.370135996Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:47:45.543491935Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T13:47:47.639561632Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:47:48.872120352Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:47:48.876494946Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:47:49.897924455Z caller=checkpoint.go:378 msg="writing series" size="642 B"
level=info ts=2026-04-28T13:47:50.98902106Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:47:50.996265369Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:47:52.63916621Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:54.252709553Z caller=checkpoint.go:378 msg="writing series" size="20 kB"
level=warn ts=2026-04-28T13:47:57.639209634Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:47:58.607415005Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=debug ts=2026-04-28T13:47:59.900373244Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:47:59.988624276Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:47:59.995747424Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:48:02.638655216Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:02.962429526Z caller=checkpoint.go:378 msg="writing series" size="184 kB"
level=debug ts=2026-04-28T13:48:04.5929288Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:48:06.336436149Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:48:07.317722065Z caller=checkpoint.go:378 msg="writing series" size="116 kB"
level=debug ts=2026-04-28T13:48:07.317747585Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-28T13:48:07.639035459Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:11.672127073Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:48:12.639593848Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:48:12.639621472Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:48:12.639629948Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:48:12.639620405Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:48:15.140698229Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53714"
level=debug ts=2026-04-28T13:48:16.026628133Z caller=checkpoint.go:378 msg="writing series" size="27 kB"
ts=2026-04-28T13:48:16.977536133Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:48754"
level=warn ts=2026-04-28T13:48:17.639357265Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:20.382349674Z caller=checkpoint.go:378 msg="writing series" size="52 kB"
level=warn ts=2026-04-28T13:48:22.639567758Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:24.736205332Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:48:27.63909055Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:29.09170846Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=info ts=2026-04-28T13:48:30.68223576Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:48:30.688382738Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:48:32.503972744Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:48:32.639255917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:33.446218699Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:48:36.338219856Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:48:37.639055928Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:37.801731586Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-28T13:48:40.003099725Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:48:40.007376091Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:48:40.007398865Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:48:40.007422235Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:48:40.010936772Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:48:40.019805969Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:48:40.021384881Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:48:42.155649919Z caller=checkpoint.go:378 msg="writing series" size="68 kB"
level=debug ts=2026-04-28T13:48:42.622237516Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:48:42.623310343Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:48:42.639186748Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:48:42.639209273Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:48:42.639218658Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:48:42.639227593Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:48:42.77655224Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:48:42.784173066Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:48:45.143249176Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53396"
level=debug ts=2026-04-28T13:48:46.511194889Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T13:48:47.639514614Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:50.866204756Z caller=checkpoint.go:378 msg="writing series" size="231 kB"
level=warn ts=2026-04-28T13:48:52.638646311Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:48:53.844081977Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:48:53.850572731Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:48:55.220830159Z caller=checkpoint.go:378 msg="writing series" size="364 kB"
level=debug ts=2026-04-28T13:48:55.220854148Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=info ts=2026-04-28T13:48:57.299909233Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:48:57.305722652Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:48:57.639366144Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:48:59.575750152Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=warn ts=2026-04-28T13:49:02.639228452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:03.930330644Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
level=debug ts=2026-04-28T13:49:03.946496665Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:49:06.339581552Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:49:07.639500825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:08.285653554Z caller=checkpoint.go:378 msg="writing series" size="568 kB"
level=info ts=2026-04-28T13:49:12.639158818Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:49:12.639187893Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:49:12.639198678Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:49:12.639188131Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:12.640261795Z caller=checkpoint.go:390 msg="flushing series" totalSize="718 kB" series=3
level=info ts=2026-04-28T13:49:12.647106011Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000007.tmp new=/var/loki/wal/checkpoint.000007
level=info ts=2026-04-28T13:49:12.648654436Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.009286644s
ts=2026-04-28T13:49:15.146175136Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:41194"
level=info ts=2026-04-28T13:49:15.874578667Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:49:15.879497386Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:49:15.879517793Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:49:15.879539984Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:49:15.882888364Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:49:16.981613721Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:53314"
level=warn ts=2026-04-28T13:49:17.638899431Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:49:17.947163904Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:49:17.951333744Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:49:19.805318245Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=warn ts=2026-04-28T13:49:22.639259147Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:49:27.638855205Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:49:30.149300851Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:49:30.155143361Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:49:32.430759226Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:49:32.639232529Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:49:32.834090663Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:49:32.842334671Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:49:36.340502511Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:49:37.639478341Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:42.622461284Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:49:42.623534089Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:49:42.638598585Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:49:42.638634325Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:49:42.638637549Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:42.63867519Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:49:42.638686995Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:49:42.638744405Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000008
level=debug ts=2026-04-28T13:49:42.639368316Z caller=checkpoint.go:378 msg="writing series" size="437 kB"
level=info ts=2026-04-28T13:49:44.260543952Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:49:44.268326154Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:49:46.983102432Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:53794"
level=debug ts=2026-04-28T13:49:46.994493314Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:49:47.63916058Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:51.349825824Z caller=checkpoint.go:378 msg="writing series" size="688 kB"
level=debug ts=2026-04-28T13:49:51.349850348Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
level=warn ts=2026-04-28T13:49:52.639466382Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:49:54.50089867Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:49:55.703575129Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:49:57.639458917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:00.059073498Z caller=checkpoint.go:378 msg="writing series" size="338 kB"
level=info ts=2026-04-28T13:50:00.922761431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:50:00.927163809Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:50:02.335937151Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:50:02.639206653Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:50:02.786581148Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:50:02.790833625Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:50:04.413952485Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T13:50:06.342757395Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:50:07.639000098Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:08.76913635Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-04-28T13:50:12.639452415Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:50:12.639477773Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:50:12.6394855Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:50:12.639503174Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:13.12316971Z caller=checkpoint.go:378 msg="writing series" size="330 kB"
level=debug ts=2026-04-28T13:50:17.478709691Z caller=checkpoint.go:378 msg="writing series" size="271 kB"
level=debug ts=2026-04-28T13:50:17.478736859Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=6
level=warn ts=2026-04-28T13:50:17.639254802Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:50:19.81745807Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:50:19.827411903Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:50:21.83350592Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-28T13:50:22.464502014Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:54840"
level=warn ts=2026-04-28T13:50:22.639206699Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:26.188234084Z caller=checkpoint.go:378 msg="writing series" size="182 kB"
level=info ts=2026-04-28T13:50:27.149002086Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:50:27.155539272Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:50:27.638881902Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:30.542735025Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
level=debug ts=2026-04-28T13:50:31.229892285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:50:32.639657338Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:34.897704603Z caller=checkpoint.go:378 msg="writing series" size="40 kB"
ts=2026-04-28T13:50:36.345909693Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:50:37.63924412Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:39.252445712Z caller=checkpoint.go:378 msg="writing series" size="3.5 kB"
level=info ts=2026-04-28T13:50:42.62310234Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:50:42.62315831Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:50:42.639549944Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:50:42.639550226Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:42.639572803Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:50:42.639580559Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:50:43.60700668Z caller=checkpoint.go:378 msg="writing series" size="561 kB"
ts=2026-04-28T13:50:45.15485177Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52082"
level=warn ts=2026-04-28T13:50:47.639341179Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:47.962302563Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=debug ts=2026-04-28T13:50:52.317344086Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=warn ts=2026-04-28T13:50:52.639630363Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:50:53.23534624Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T13:50:54.150709868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:50:54.156497785Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:50:56.551655317Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:50:56.681765394Z caller=checkpoint.go:378 msg="writing series" size="9.8 MB"
level=debug ts=2026-04-28T13:50:56.681787507Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=9
level=info ts=2026-04-28T13:50:57.032859155Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:50:57.036822612Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:50:57.639387027Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:01.026696667Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:51:02.63953295Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:05.381843264Z caller=checkpoint.go:378 msg="writing series" size="66 kB"
ts=2026-04-28T13:51:06.349448596Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:51:07.639030389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:09.736629947Z caller=checkpoint.go:378 msg="writing series" size="251 kB"
level=info ts=2026-04-28T13:51:12.639144002Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:51:12.639155695Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:12.639171448Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:51:12.639179484Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:51:14.091421685Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
ts=2026-04-28T13:51:15.158038522Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:53322"
level=info ts=2026-04-28T13:51:15.697873788Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:51:15.704526488Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:51:16.276092195Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:51:16.282715257Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:51:16.989188901Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:58736"
level=warn ts=2026-04-28T13:51:17.639258925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:18.44630975Z caller=checkpoint.go:378 msg="writing series" size="201 kB"
ts=2026-04-28T13:51:22.468811438Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:39698"
level=warn ts=2026-04-28T13:51:22.6394251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:22.800737466Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T13:51:27.155263256Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=debug ts=2026-04-28T13:51:27.598722561Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:51:27.639210571Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:29.966061403Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:51:31.510296313Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:51:32.639464575Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:35.865612039Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
ts=2026-04-28T13:51:36.353020039Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:51:37.638816361Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:40.219876464Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T13:51:42.62306924Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:51:42.623078748Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:51:42.638633815Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:51:42.638656297Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T13:51:42.638649208Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:51:42.638682012Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:51:44.575485316Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
ts=2026-04-28T13:51:45.161014285Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:56394"
level=info ts=2026-04-28T13:51:47.015533567Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:51:47.020093644Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:51:47.639540908Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:51:47.775867335Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:51:47.780082617Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:51:48.930321253Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-28T13:51:52.470221851Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60618"
level=warn ts=2026-04-28T13:51:52.639148366Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:53.28473073Z caller=checkpoint.go:378 msg="writing series" size="137 kB"
level=debug ts=2026-04-28T13:51:55.663680092Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:51:57.639130273Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:51:57.639170905Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:51:57.639180731Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=14
level=debug ts=2026-04-28T13:52:01.994600063Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=warn ts=2026-04-28T13:52:02.638855389Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:52:05.998348948Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:52:06.009331639Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:52:06.009358791Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:52:06.009386729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:52:06.016195637Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:52:06.349105987Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-04-28T13:52:06.355365373Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=info ts=2026-04-28T13:52:06.640944431Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:52:06.646274367Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:52:07.63915105Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:10.70454111Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:52:12.639273682Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:52:12.639297164Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:52:12.639304979Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:52:12.63930842Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:15.058689629Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:52:17.639198574Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:19.41341533Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=debug ts=2026-04-28T13:52:20.148704586Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:52:22.190453329Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:52:22.192243027Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=warn ts=2026-04-28T13:52:22.639430522Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:23.768390689Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-28T13:52:24.029849791Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:52:24.037294334Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:52:27.639374128Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:28.12389602Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=info ts=2026-04-28T13:52:29.351378298Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:52:29.355541705Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:52:32.479860567Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-28T13:52:32.479885702Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.5 MB" series=8
level=warn ts=2026-04-28T13:52:32.63864939Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:52:36.356229647Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:52:36.83351003Z caller=checkpoint.go:378 msg="writing series" size="162 kB"
level=warn ts=2026-04-28T13:52:37.639227155Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:41.18801298Z caller=checkpoint.go:378 msg="writing series" size="452 kB"
level=info ts=2026-04-28T13:52:42.622977173Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:52:42.623306699Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:52:42.638612933Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:52:42.638640301Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:52:42.638649942Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:52:42.638671315Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:44.728138696Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:52:45.167547593Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:60668"
level=debug ts=2026-04-28T13:52:45.543025385Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
level=warn ts=2026-04-28T13:52:47.639079423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:52:47.73838969Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:52:47.74414421Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:52:49.897163951Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=warn ts=2026-04-28T13:52:52.638712672Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:54.252703515Z caller=checkpoint.go:378 msg="writing series" size="21 kB"
level=warn ts=2026-04-28T13:52:57.639097775Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:52:58.607445851Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T13:53:02.638641958Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:53:02.960646022Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:53:02.961769104Z caller=checkpoint.go:378 msg="writing series" size="212 kB"
level=debug ts=2026-04-28T13:53:02.961786524Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=7
level=error ts=2026-04-28T13:53:02.966803192Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:53:02.978903473Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:53:02.985648253Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
ts=2026-04-28T13:53:06.358528581Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T13:53:07.316860963Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-28T13:53:07.638705032Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:10.191767856Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:53:11.672153078Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:53:12.639291022Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:53:12.639314719Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:53:12.639322313Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:53:12.639334771Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:16.026184329Z caller=checkpoint.go:378 msg="writing series" size="30 kB"
ts=2026-04-28T13:53:17.000645599Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:43526"
level=warn ts=2026-04-28T13:53:17.639291788Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:18.097915588Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:53:20.381764117Z caller=checkpoint.go:378 msg="writing series" size="80 kB"
level=warn ts=2026-04-28T13:53:22.638769555Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:24.736779452Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:53:27.639463476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:53:28.201331755Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:53:28.205859207Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:53:29.091339073Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T13:53:32.639396251Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:33.445704857Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=debug ts=2026-04-28T13:53:35.785239374Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:53:36.361975042Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:53:37.639624962Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:37.801072602Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-28T13:53:40.341063491Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:53:40.348071398Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:53:42.156097731Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=info ts=2026-04-28T13:53:42.622936681Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:53:42.622980238Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:53:42.638969227Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:53:42.63899084Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:53:42.639002544Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:53:42.639028207Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:46.510718648Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T13:53:47.639488029Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:53:48.996827373Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=3 total_comp="7.4 MB" avg_comp="2.5 MB" total_uncomp="65 MB" avg_uncomp="22 MB" full=2 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:53:49.00452752Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 3, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:53:50.865815578Z caller=checkpoint.go:378 msg="writing series" size="279 kB"
level=warn ts=2026-04-28T13:53:52.639113452Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:55.220603596Z caller=checkpoint.go:378 msg="writing series" size="441 kB"
level=debug ts=2026-04-28T13:53:55.220627752Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.4 MB" series=12
level=info ts=2026-04-28T13:53:57.398047947Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:53:57.403804926Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:53:57.403826548Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:53:57.403850753Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:53:57.408808642Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:53:57.639302579Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:53:59.575314388Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-28T13:54:01.039635994Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:54:02.639655113Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:54:03.211681987Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:54:03.929906669Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:54:06.364377719Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:54:07.639549825Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:54:08.285423189Z caller=checkpoint.go:378 msg="writing series" size="594 kB"
level=warn ts=2026-04-28T13:54:12.638999343Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:54:12.639014808Z caller=checkpoint.go:390 msg="flushing series" totalSize="744 kB" series=3
level=info ts=2026-04-28T13:54:12.639087304Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:54:12.639105732Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:54:12.639113704Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:54:12.645611669Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000008.tmp new=/var/loki/wal/checkpoint.000008
level=info ts=2026-04-28T13:54:12.647114826Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008169381s
ts=2026-04-28T13:54:15.176721883Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:52438"
level=info ts=2026-04-28T13:54:17.079672934Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:54:17.086784868Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:54:17.639199037Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:54:17.673324314Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:54:17.678120545Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:54:22.639171102Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:54:27.6396877Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:54:28.362148348Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:54:30.879520979Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:54:30.887923151Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:54:32.639370784Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T13:54:36.366152956Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:54:37.638649456Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:54:42.622984003Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:54:42.623308323Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:54:42.63892675Z caller=checkpoint.go:612 msg="starting checkpoint"
level=warn ts=2026-04-28T13:54:42.638931443Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:54:42.638939805Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:54:42.638956843Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:54:42.638965376Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:54:42.639065578Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000009
level=debug ts=2026-04-28T13:54:42.639765094Z caller=checkpoint.go:378 msg="writing series" size="509 kB"
level=info ts=2026-04-28T13:54:43.429911957Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:54:43.438921679Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:54:46.994445644Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:54:47.639553436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:54:49.671092633Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:54:51.350019565Z caller=checkpoint.go:378 msg="writing series" size="814 kB"
level=debug ts=2026-04-28T13:54:51.350042184Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.3 MB" series=3
level=warn ts=2026-04-28T13:54:52.639146552Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:54:52.966216232Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:54:52.971363176Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:54:55.704806357Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:54:57.638764665Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:00.059012698Z caller=checkpoint.go:378 msg="writing series" size="473 kB"
level=debug ts=2026-04-28T13:55:02.326836285Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:55:02.639447236Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:04.414469734Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=info ts=2026-04-28T13:55:05.33265543Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:55:05.340386654Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:55:06.368355785Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:55:07.638703573Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:08.768492983Z caller=checkpoint.go:378 msg="writing series" size="188 kB"
level=info ts=2026-04-28T13:55:12.639598027Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:55:12.639625116Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:55:12.639634707Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:55:12.639646864Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:13.124265531Z caller=checkpoint.go:378 msg="writing series" size="167 kB"
level=debug ts=2026-04-28T13:55:17.4789925Z caller=checkpoint.go:378 msg="writing series" size="94 kB"
level=warn ts=2026-04-28T13:55:17.639434011Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:21.833278316Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
ts=2026-04-28T13:55:22.484904128Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:60410"
level=warn ts=2026-04-28T13:55:22.639430336Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:55:24.020359775Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:55:24.024599611Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:55:24.024618201Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:55:24.024639687Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:55:24.028157981Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:55:25.774551379Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:55:26.187996028Z caller=checkpoint.go:378 msg="writing series" size="195 kB"
level=debug ts=2026-04-28T13:55:26.188019723Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=8
level=warn ts=2026-04-28T13:55:27.63924787Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:30.543457088Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=warn ts=2026-04-28T13:55:32.639031979Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:34.897442504Z caller=checkpoint.go:378 msg="writing series" size="42 kB"
level=debug ts=2026-04-28T13:55:35.69589071Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:55:36.369962106Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=info ts=2026-04-28T13:55:37.080648772Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:55:37.089052501Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:55:37.639464015Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:55:38.410247279Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:55:38.416850525Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:55:39.253162123Z caller=checkpoint.go:378 msg="writing series" size="3.7 kB"
level=info ts=2026-04-28T13:55:42.622976554Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:55:42.623006122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:55:42.639442672Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:55:42.639464738Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:55:42.63947236Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:55:42.63947874Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:43.607610554Z caller=checkpoint.go:378 msg="writing series" size="561 kB"
level=info ts=2026-04-28T13:55:44.938309892Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:55:44.943570749Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
ts=2026-04-28T13:55:45.185872267Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:39974"
ts=2026-04-28T13:55:47.011778745Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:41146"
level=warn ts=2026-04-28T13:55:47.638952987Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:55:47.962549562Z caller=checkpoint.go:378 msg="writing series" size="6.1 kB"
level=debug ts=2026-04-28T13:55:52.316850798Z caller=checkpoint.go:378 msg="writing series" size="122 kB"
level=warn ts=2026-04-28T13:55:52.639177159Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:55:52.921395481Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:55:52.926954981Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:55:52.926980309Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:55:52.927003942Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:55:52.930379066Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:55:56.679704984Z caller=checkpoint.go:378 msg="writing series" size="10 MB"
level=debug ts=2026-04-28T13:55:56.679729143Z caller=checkpoint.go:390 msg="flushing series" totalSize="11 MB" series=7
level=warn ts=2026-04-28T13:55:57.639057611Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:00.366798412Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:56:00.368456208Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:56:01.026602652Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T13:56:02.638655679Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:05.381303107Z caller=checkpoint.go:378 msg="writing series" size="88 kB"
ts=2026-04-28T13:56:06.371711938Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T13:56:07.639574119Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:09.736466328Z caller=checkpoint.go:378 msg="writing series" size="41 kB"
level=debug ts=2026-04-28T13:56:10.868841869Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=info ts=2026-04-28T13:56:11.569696299Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:56:11.575191306Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:56:12.63895846Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:56:12.638974953Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:56:12.638993029Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:56:12.639000129Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:56:12.658122672Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:56:12.66276787Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:56:14.091788419Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
ts=2026-04-28T13:56:15.188603538Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58796"
ts=2026-04-28T13:56:17.013257775Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:47098"
level=warn ts=2026-04-28T13:56:17.63941799Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:18.446815124Z caller=checkpoint.go:378 msg="writing series" size="232 kB"
level=warn ts=2026-04-28T13:56:22.638641698Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:22.801079497Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=info ts=2026-04-28T13:56:26.412254536Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:56:26.421500419Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:56:27.156437746Z caller=checkpoint.go:378 msg="writing series" size="8.4 kB"
level=warn ts=2026-04-28T13:56:27.638682071Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:31.510655746Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T13:56:32.639205416Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:56:32.773543565Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:56:32.778991686Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:56:35.865302919Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
ts=2026-04-28T13:56:36.372803769Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:56:37.354169094Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:56:37.638793669Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:40.220470136Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=debug ts=2026-04-28T13:56:42.622568699Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:56:42.623617334Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-28T13:56:42.638793801Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:56:42.638812216Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:56:42.638825326Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:56:42.638843823Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:56:44.574863954Z caller=checkpoint.go:378 msg="writing series" size="179 kB"
level=info ts=2026-04-28T13:56:45.776594386Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:56:45.784837826Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:56:47.63962333Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:48.929845583Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
ts=2026-04-28T13:56:52.489552322Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:40776"
level=warn ts=2026-04-28T13:56:52.639340835Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:53.052796309Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:56:53.28548698Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=info ts=2026-04-28T13:56:53.311535212Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:56:53.315915694Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:56:57.63909176Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:56:57.640216595Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=debug ts=2026-04-28T13:57:01.994353506Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T13:57:02.639287404Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:04.250593381Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:57:06.349752143Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
ts=2026-04-28T13:57:06.374085905Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:57:07.639334609Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:10.704350545Z caller=checkpoint.go:378 msg="writing series" size="1.5 kB"
level=info ts=2026-04-28T13:57:12.638644044Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:57:12.638685228Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:57:12.638699087Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:57:12.638697257Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:15.058776614Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T13:57:17.639056214Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:19.414311226Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-28T13:57:19.636248989Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:57:19.647784688Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:57:22.639622296Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:23.768582936Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=info ts=2026-04-28T13:57:24.890952263Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:57:24.896355345Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=warn ts=2026-04-28T13:57:27.638934516Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:28.12357544Z caller=checkpoint.go:378 msg="writing series" size="7.8 kB"
level=debug ts=2026-04-28T13:57:31.421142627Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:57:32.478781599Z caller=checkpoint.go:378 msg="writing series" size="1.4 MB"
level=debug ts=2026-04-28T13:57:32.478803904Z caller=checkpoint.go:390 msg="flushing series" totalSize="2.5 MB" series=22
level=warn ts=2026-04-28T13:57:32.639423971Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:35.880166164Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:57:36.375627131Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=debug ts=2026-04-28T13:57:36.833732351Z caller=checkpoint.go:378 msg="writing series" size="174 kB"
level=warn ts=2026-04-28T13:57:37.639325118Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:57:38.351715535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:57:38.356978776Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T13:57:41.188837666Z caller=checkpoint.go:378 msg="writing series" size="375 kB"
level=debug ts=2026-04-28T13:57:42.622548209Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:57:42.62360861Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=warn ts=2026-04-28T13:57:42.639005303Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:57:42.639005669Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:57:42.639025735Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:57:42.639033525Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
ts=2026-04-28T13:57:45.198432221Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:57580"
level=debug ts=2026-04-28T13:57:45.542845582Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
ts=2026-04-28T13:57:47.020644847Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:60320"
level=warn ts=2026-04-28T13:57:47.63963436Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:57:49.262644526Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:57:49.268375668Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:57:49.898152846Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=warn ts=2026-04-28T13:57:52.639263061Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:54.252907058Z caller=checkpoint.go:378 msg="writing series" size="22 kB"
level=warn ts=2026-04-28T13:57:57.639056989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:57:58.207248495Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T13:57:58.607375963Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=info ts=2026-04-28T13:58:00.258378169Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:58:00.267301247Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T13:58:02.63879377Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:02.962746557Z caller=checkpoint.go:378 msg="writing series" size="199 kB"
ts=2026-04-28T13:58:06.376903793Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=debug ts=2026-04-28T13:58:07.317432752Z caller=checkpoint.go:378 msg="writing series" size="129 kB"
level=debug ts=2026-04-28T13:58:07.317458005Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-04-28T13:58:07.639733628Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:11.671795822Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T13:58:12.638851436Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:58:12.638882083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:58:12.638892645Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:58:12.638902989Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:16.02745461Z caller=checkpoint.go:378 msg="writing series" size="32 kB"
ts=2026-04-28T13:58:17.022367408Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46938"
level=warn ts=2026-04-28T13:58:17.63915172Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:58:18.661715643Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:58:18.668025293Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T13:58:20.38178713Z caller=checkpoint.go:378 msg="writing series" size="113 kB"
level=warn ts=2026-04-28T13:58:22.638938857Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:23.015615934Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T13:58:24.736620481Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=debug ts=2026-04-28T13:58:27.316843946Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:58:27.639439567Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:29.091147796Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T13:58:32.63928977Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:33.445874423Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
ts=2026-04-28T13:58:36.378105913Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=info ts=2026-04-28T13:58:37.376716544Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:58:37.391473444Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T13:58:37.63908952Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:37.801520108Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=info ts=2026-04-28T13:58:40.751377518Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:58:40.757631258Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:58:40.757652842Z caller=flush.go:240 component=ingester loop=14 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T13:58:40.757696034Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:58:40.763461701Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:58:41.9327805Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:58:41.936971919Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T13:58:42.156330209Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-28T13:58:42.62226187Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T13:58:42.623298425Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T13:58:42.639176373Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:58:42.639192392Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:58:42.639201079Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T13:58:42.639223804Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:46.510706362Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
ts=2026-04-28T13:58:47.023603348Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:49032"
level=warn ts=2026-04-28T13:58:47.638717513Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:50.865592428Z caller=checkpoint.go:378 msg="writing series" size="95 kB"
level=warn ts=2026-04-28T13:58:52.638951925Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:58:53.71803729Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:58:53.729292088Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T13:58:55.22035891Z caller=checkpoint.go:378 msg="writing series" size="509 kB"
level=debug ts=2026-04-28T13:58:55.220381989Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=11
level=warn ts=2026-04-28T13:58:57.638965916Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:58:59.575219219Z caller=checkpoint.go:378 msg="writing series" size="96 kB"
level=debug ts=2026-04-28T13:59:00.340631037Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T13:59:02.639650833Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:03.930035242Z caller=checkpoint.go:378 msg="writing series" size="54 kB"
ts=2026-04-28T13:59:06.380317629Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T13:59:07.639067499Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:08.285043829Z caller=checkpoint.go:378 msg="writing series" size="623 kB"
level=info ts=2026-04-28T13:59:12.638639566Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T13:59:12.638639868Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:12.63868083Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:59:12.638689644Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T13:59:12.639684315Z caller=checkpoint.go:390 msg="flushing series" totalSize="773 kB" series=3
level=info ts=2026-04-28T13:59:12.64629874Z caller=checkpoint.go:499 msg="atomic checkpoint finished" old=/var/loki/wal/checkpoint.000009.tmp new=/var/loki/wal/checkpoint.000009
level=info ts=2026-04-28T13:59:12.647977908Z caller=checkpoint.go:570 msg="checkpoint done" time=4m30.008776886s
level=debug ts=2026-04-28T13:59:13.102624237Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T13:59:15.207772167Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:59796"
ts=2026-04-28T13:59:17.025207149Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:46638"
level=warn ts=2026-04-28T13:59:17.638930024Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:59:18.492790278Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T13:59:18.498227182Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T13:59:21.926705535Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T13:59:21.930877241Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
ts=2026-04-28T13:59:22.500945259Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:34210"
level=warn ts=2026-04-28T13:59:22.639181564Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=warn ts=2026-04-28T13:59:27.639590969Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:59:29.228628317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T13:59:29.236248501Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=info ts=2026-04-28T13:59:32.140211522Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T13:59:32.144552191Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T13:59:32.639408088Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:34.399968173Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T13:59:36.382513021Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T13:59:37.63920154Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:59:42.623113256Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T13:59:42.623915122Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=warn ts=2026-04-28T13:59:42.639268878Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T13:59:42.639280815Z caller=checkpoint.go:612 msg="starting checkpoint"
level=info ts=2026-04-28T13:59:42.639291957Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T13:59:42.639308007Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T13:59:42.639316742Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=info ts=2026-04-28T13:59:42.639402098Z caller=checkpoint.go:337 msg="attempting checkpoint for" dir=/var/loki/wal/checkpoint.000010
level=debug ts=2026-04-28T13:59:42.639886415Z caller=checkpoint.go:378 msg="writing series" size="369 kB"
level=debug ts=2026-04-28T13:59:46.994814493Z caller=checkpoint.go:378 msg="writing series" size="5.4 kB"
level=warn ts=2026-04-28T13:59:47.63914192Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:51.350072362Z caller=checkpoint.go:378 msg="writing series" size="689 kB"
level=debug ts=2026-04-28T13:59:51.350091281Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=3
ts=2026-04-28T13:59:52.50254308Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:45080"
level=warn ts=2026-04-28T13:59:52.638674547Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T13:59:53.17184763Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=error ts=2026-04-28T13:59:53.173637284Z caller=reporter.go:247 msg="failed to delete corrupted cluster seed file, deleting it" err="operation error S3: DeleteObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Delete \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=DeleteObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host"
level=debug ts=2026-04-28T13:59:55.704689626Z caller=checkpoint.go:378 msg="writing series" size="2.0 kB"
level=warn ts=2026-04-28T13:59:57.639716385Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:00.060466565Z caller=checkpoint.go:378 msg="writing series" size="370 kB"
level=debug ts=2026-04-28T14:00:02.526110118Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=warn ts=2026-04-28T14:00:02.639339378Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:04.414036766Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
ts=2026-04-28T14:00:06.383826368Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T14:00:07.639606687Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:00:07.906010517Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:00:07.912503224Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T14:00:08.769124102Z caller=checkpoint.go:378 msg="writing series" size="189 kB"
level=info ts=2026-04-28T14:00:11.380810451Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:00:11.385548415Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T14:00:12.638705819Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:00:12.638734948Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:00:12.638758239Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:00:12.63877473Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T14:00:13.124258946Z caller=checkpoint.go:378 msg="writing series" size="241 kB"
level=info ts=2026-04-28T14:00:14.703783191Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:00:14.70836193Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-28T14:00:17.027122079Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:34842"
level=debug ts=2026-04-28T14:00:17.478795196Z caller=checkpoint.go:378 msg="writing series" size="151 kB"
level=warn ts=2026-04-28T14:00:17.63885831Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:21.833560861Z caller=checkpoint.go:378 msg="writing series" size="5.3 kB"
level=warn ts=2026-04-28T14:00:22.638855097Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:00:22.973710884Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T14:00:22.982768616Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T14:00:26.188865181Z caller=checkpoint.go:378 msg="writing series" size="223 kB"
level=debug ts=2026-04-28T14:00:26.188888417Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=8
level=warn ts=2026-04-28T14:00:27.639582453Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:28.204453782Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:00:30.543824591Z caller=checkpoint.go:378 msg="writing series" size="75 kB"
level=warn ts=2026-04-28T14:00:32.639261565Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:34.898411703Z caller=checkpoint.go:378 msg="writing series" size="43 kB"
level=debug ts=2026-04-28T14:00:34.918781799Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
ts=2026-04-28T14:00:36.386147361Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=warn ts=2026-04-28T14:00:37.639605697Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:39.253307466Z caller=checkpoint.go:378 msg="writing series" size="4.0 kB"
level=debug ts=2026-04-28T14:00:42.62235308Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T14:00:42.623386087Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T14:00:42.623599876Z caller=manager.go:249 index-store=tsdb-2024-04-01 component=tsdb-manager msg="building heads"
level=debug ts=2026-04-28T14:00:42.623639699Z caller=head_manager.go:440 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="listed WALs" pd=1974871 n=1
level=debug ts=2026-04-28T14:00:42.623699124Z caller=head_manager.go:445 index-store=tsdb-2024-04-01 component=tsdb-head-manager msg="removing wals" pd=1974871 n=1
level=warn ts=2026-04-28T14:00:42.638908867Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:00:42.638942736Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:00:42.638963328Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:00:42.63897097Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T14:00:43.608738073Z caller=checkpoint.go:378 msg="writing series" size="543 kB"
ts=2026-04-28T14:00:45.216524621Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:36378"
ts=2026-04-28T14:00:47.028486754Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:54174"
level=warn ts=2026-04-28T14:00:47.639627314Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:00:47.963250766Z caller=checkpoint.go:378 msg="writing series" size="6.7 kB"
level=debug ts=2026-04-28T14:00:52.318010613Z caller=checkpoint.go:378 msg="writing series" size="130 kB"
ts=2026-04-28T14:00:52.507227944Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.73:58936"
level=warn ts=2026-04-28T14:00:52.638636796Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:00:54.349771296Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T14:00:54.352613826Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:00:54.356288185Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=7 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:00:54.356300361Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:00:54.356316387Z caller=flush.go:240 component=ingester loop=21 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T14:00:54.356337474Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:00:54.360651235Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=info ts=2026-04-28T14:00:56.439851113Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:00:56.444642078Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T14:00:56.682324033Z caller=checkpoint.go:378 msg="writing series" size="11 MB"
level=debug ts=2026-04-28T14:00:56.682348017Z caller=checkpoint.go:390 msg="flushing series" totalSize="12 MB" series=7
level=warn ts=2026-04-28T14:00:57.63959776Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:00.267192463Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:01:01.027586353Z caller=checkpoint.go:378 msg="writing series" size="3.8 kB"
level=warn ts=2026-04-28T14:01:02.638869651Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:01:04.811705868Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:01:04.820235146Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T14:01:05.38165095Z caller=checkpoint.go:378 msg="writing series" size="114 kB"
ts=2026-04-28T14:01:06.388598375Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T14:01:07.63958917Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:09.737273941Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=info ts=2026-04-28T14:01:12.639009668Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:01:12.639039445Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:01:12.639047079Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T14:01:12.639062734Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:14.091201707Z caller=checkpoint.go:378 msg="writing series" size="18 kB"
level=info ts=2026-04-28T14:01:14.360042453Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T14:01:14.367836139Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T14:01:17.639302569Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:18.446291005Z caller=checkpoint.go:378 msg="writing series" size="262 kB"
level=warn ts=2026-04-28T14:01:22.638859269Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:22.801264768Z caller=checkpoint.go:378 msg="writing series" size="12 kB"
level=debug ts=2026-04-28T14:01:27.15633207Z caller=checkpoint.go:378 msg="writing series" size="7.5 kB"
level=warn ts=2026-04-28T14:01:27.639481863Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:01:28.219397024Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:01:28.229097934Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T14:01:31.51158271Z caller=checkpoint.go:378 msg="writing series" size="1.7 kB"
level=warn ts=2026-04-28T14:01:32.639272063Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:33.933642854Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T14:01:35.104900392Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:01:35.109065429Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=8 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T14:01:35.805272436Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:01:35.865696384Z caller=checkpoint.go:378 msg="writing series" size="2.7 kB"
ts=2026-04-28T14:01:36.390379786Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-compactor-0-00342869 10.128.0.73:7946"
level=warn ts=2026-04-28T14:01:37.638827013Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:40.220640788Z caller=checkpoint.go:378 msg="writing series" size="3.2 kB"
level=info ts=2026-04-28T14:01:42.62303429Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=debug ts=2026-04-28T14:01:42.623624909Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=warn ts=2026-04-28T14:01:42.639455981Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:01:42.639463806Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:01:42.639486935Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:01:42.639497295Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T14:01:44.576160729Z caller=checkpoint.go:378 msg="writing series" size="200 kB"
level=info ts=2026-04-28T14:01:46.453428091Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:01:46.460391916Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:01:46.460418349Z caller=flush.go:240 component=ingester loop=26 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T14:01:46.460445849Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:01:46.464453815Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=warn ts=2026-04-28T14:01:47.639266259Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:48.930718525Z caller=checkpoint.go:378 msg="writing series" size="140 kB"
level=warn ts=2026-04-28T14:01:52.638709476Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:53.284942463Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T14:01:57.639413354Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:01:57.6404956Z caller=checkpoint.go:378 msg="writing series" size="57 kB"
level=info ts=2026-04-28T14:02:01.973310845Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:02:01.978822542Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=debug ts=2026-04-28T14:02:01.994943134Z caller=checkpoint.go:378 msg="writing series" size="14 kB"
level=warn ts=2026-04-28T14:02:02.639561622Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:06.349749301Z caller=checkpoint.go:378 msg="writing series" size="67 kB"
level=debug ts=2026-04-28T14:02:06.349777901Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.1 MB" series=16
ts=2026-04-28T14:02:06.392119424Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T14:02:07.639574968Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:02:08.61951915Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T14:02:08.628409104Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=5 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=debug ts=2026-04-28T14:02:10.59941625Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:02:10.705107898Z caller=checkpoint.go:378 msg="writing series" size="4.3 kB"
level=info ts=2026-04-28T14:02:12.638965959Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=warn ts=2026-04-28T14:02:12.63899074Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:12.639004314Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:02:12.63901872Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T14:02:15.059458639Z caller=checkpoint.go:378 msg="writing series" size="3.1 kB"
level=warn ts=2026-04-28T14:02:17.639623474Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:19.414300901Z caller=checkpoint.go:378 msg="writing series" size="31 kB"
level=info ts=2026-04-28T14:02:20.193652317Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:02:20.201153166Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=9 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:02:20.201179115Z caller=flush.go:240 component=ingester loop=15 org_id=kubearchive msg="failed to flush" err="terminated after 10 retries"
level=info ts=2026-04-28T14:02:20.201204816Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:02:20.205250295Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=0 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=warn ts=2026-04-28T14:02:22.639524937Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:02:22.814968439Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:02:22.82038576Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T14:02:23.768688348Z caller=checkpoint.go:378 msg="writing series" size="5.9 kB"
level=warn ts=2026-04-28T14:02:27.639394545Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:28.123808504Z caller=checkpoint.go:378 msg="writing series" size="8.0 kB"
level=debug ts=2026-04-28T14:02:28.233376276Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=debug ts=2026-04-28T14:02:32.480173544Z caller=checkpoint.go:378 msg="writing series" size="1.5 MB"
level=debug ts=2026-04-28T14:02:32.48019517Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.6 MB" series=6
level=warn ts=2026-04-28T14:02:32.638921947Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T14:02:36.394586431Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T14:02:36.833788212Z caller=checkpoint.go:378 msg="writing series" size="187 kB"
level=warn ts=2026-04-28T14:02:37.639672704Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:02:37.819603001Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:02:37.825651419Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
level=info ts=2026-04-28T14:02:38.074945459Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:02:38.079306292Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=1 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T14:02:39.686576292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:02:41.189037251Z caller=checkpoint.go:378 msg="writing series" size="464 kB"
level=debug ts=2026-04-28T14:02:42.622412458Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T14:02:42.623483121Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T14:02:42.639105152Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:02:42.639129378Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:02:42.639137882Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T14:02:42.639262855Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
ts=2026-04-28T14:02:45.229144129Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:58486"
level=debug ts=2026-04-28T14:02:45.543195823Z caller=checkpoint.go:378 msg="writing series" size="244 kB"
ts=2026-04-28T14:02:47.036885881Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.128.0.71:59500"
level=warn ts=2026-04-28T14:02:47.63888769Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:49.897976463Z caller=checkpoint.go:378 msg="writing series" size="657 B"
level=info ts=2026-04-28T14:02:52.079954571Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=ec4fff4ad1321f8e immediate=false num_chunks=4 total_comp="10 MB" avg_comp="2.5 MB" total_uncomp="89 MB" avg_uncomp="22 MB" full=3 synced=1 labels="{stream=\"openshift-gitops\"}"
level=error ts=2026-04-28T14:02:52.089775305Z caller=flush.go:262 component=ingester loop=14 org_id=kubearchive msg="failed to flush" retries=6 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/ec4fff4ad1321f8e/19dd43128a7%3A19dd431f27a%3A38dd5ae2?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 4, labels: {stream=\"openshift-gitops\"}"
level=warn ts=2026-04-28T14:02:52.638891655Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:54.252654175Z caller=checkpoint.go:378 msg="writing series" size="23 kB"
level=warn ts=2026-04-28T14:02:57.639139881Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:02:58.608234833Z caller=checkpoint.go:378 msg="writing series" size="17 kB"
level=warn ts=2026-04-28T14:03:02.639675983Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:02.962616378Z caller=checkpoint.go:378 msg="writing series" size="226 kB"
level=debug ts=2026-04-28T14:03:02.9626394Z caller=checkpoint.go:390 msg="flushing series" totalSize="1.2 MB" series=7
ts=2026-04-28T14:03:06.397991111Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-querier-cc79b8c59-f44f5-461269a2 10.130.0.67:7946"
level=debug ts=2026-04-28T14:03:07.317965491Z caller=checkpoint.go:378 msg="writing series" size="135 kB"
level=warn ts=2026-04-28T14:03:07.639242929Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:09.426150181Z caller=reporter.go:244 msg="failed to read cluster seed file" err="failed to get s3 object: operation error S3: GetObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Get \"http://minio:9000/loki-data/loki_cluster_seed.json?x-id=GetObject\": 3 errors occurred:\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\t* dial tcp: lookup minio on 172.30.0.10:53: no such host\n\n\nfailed to get rate limit token, retry quota exceeded, 0 available, 5 requested"
level=info ts=2026-04-28T14:03:10.209374754Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=5a21c384f089c4b5 immediate=false num_chunks=1 total_comp="1.2 MB" avg_comp="1.2 MB" total_uncomp="6.8 MB" avg_uncomp="6.8 MB" synced=1 labels="{stream=\"openshift-ovn-kubernetes\"}"
level=error ts=2026-04-28T14:03:10.21422729Z caller=flush.go:262 component=ingester loop=21 org_id=kubearchive msg="failed to flush" retries=4 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/5a21c384f089c4b5/19dd431233c%3A19dd43a5785%3A11130862?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-ovn-kubernetes\"}"
level=debug ts=2026-04-28T14:03:11.672501364Z caller=checkpoint.go:378 msg="writing series" size="7.0 kB"
level=info ts=2026-04-28T14:03:12.638768641Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:03:12.638798219Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=info ts=2026-04-28T14:03:12.638808436Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=warn ts=2026-04-28T14:03:12.638806642Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:03:12.867723483Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=dbeb2421470a9a4f immediate=false num_chunks=1 total_comp="482 kB" avg_comp="482 kB" total_uncomp="5.0 MB" avg_uncomp="5.0 MB" synced=1 labels="{stream=\"openshift-operators\"}"
level=error ts=2026-04-28T14:03:12.872400592Z caller=flush.go:262 component=ingester loop=15 org_id=kubearchive msg="failed to flush" retries=2 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/dbeb2421470a9a4f/19dd4312a97%3A19dd4368dc3%3A26bf5c81?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-operators\"}"
level=debug ts=2026-04-28T14:03:13.130633682Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
ts=2026-04-28T14:03:15.232220724Z caller=memberlist_logger.go:74 level=debug msg="Stream connection from=10.130.0.67:54010"
level=debug ts=2026-04-28T14:03:16.026903202Z caller=checkpoint.go:378 msg="writing series" size="36 kB"
level=warn ts=2026-04-28T14:03:17.639081736Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:20.382273055Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T14:03:22.638972595Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:24.737393234Z caller=checkpoint.go:378 msg="writing series" size="13 kB"
level=warn ts=2026-04-28T14:03:27.639637359Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:29.092029216Z caller=checkpoint.go:378 msg="writing series" size="143 kB"
level=warn ts=2026-04-28T14:03:32.63868872Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:33.446068942Z caller=checkpoint.go:378 msg="writing series" size="25 kB"
level=info ts=2026-04-28T14:03:34.562310859Z caller=flush.go:305 component=ingester msg="flushing stream" user=kubearchive fp=1b9a6a58553f1a9a immediate=false num_chunks=1 total_comp="479 kB" avg_comp="479 kB" total_uncomp="3.8 MB" avg_uncomp="3.8 MB" synced=1 labels="{stream=\"openshift-deployment-validation-operator\"}"
level=error ts=2026-04-28T14:03:34.567985535Z caller=flush.go:262 component=ingester loop=26 org_id=kubearchive msg="failed to flush" retries=3 err="failed to flush chunks: store put chunk: operation error S3: PutObject, https response error StatusCode: 0, RequestID: , HostID: , request send failed, Put \"http://minio:9000/loki-data/kubearchive/1b9a6a58553f1a9a/19dd431238f%3A19dd4529c18%3Ad8b9455e?x-id=PutObject\": dial tcp: lookup minio on 172.30.0.10:53: no such host, num_chunks: 1, labels: {stream=\"openshift-deployment-validation-operator\"}"
ts=2026-04-28T14:03:36.399954339Z caller=memberlist_logger.go:74 level=debug msg="Initiating push/pull sync with: loki-distributor-5d4786f4bc-l5mk4-847f1eb5 10.128.0.71:7946"
level=warn ts=2026-04-28T14:03:37.638795423Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=debug ts=2026-04-28T14:03:37.801273055Z caller=checkpoint.go:378 msg="writing series" size="108 kB"
level=debug ts=2026-04-28T14:03:41.613774292Z caller=flush.go:215 msg="computed flush rate" rate=3.3333333333333335
level=debug ts=2026-04-28T14:03:42.156572766Z caller=checkpoint.go:378 msg="writing series" size="69 kB"
level=debug ts=2026-04-28T14:03:42.622603331Z caller=memcached_client_selector.go:109 msg="updating servers" servers=10.130.0.71:11211 count=1
level=info ts=2026-04-28T14:03:42.623688289Z caller=table_manager.go:136 index-store=tsdb-2024-04-01 msg="uploading tables"
level=info ts=2026-04-28T14:03:42.63863046Z caller=recalculate_owned_streams.go:49 msg="starting recalculate owned streams job"
level=debug ts=2026-04-28T14:03:42.63867963Z caller=recalculate_owned_streams.go:60 msg="ring is not changed, skipping the job"
level=warn ts=2026-04-28T14:03:42.638673524Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"
level=info ts=2026-04-28T14:03:42.638688665Z caller=recalculate_owned_streams.go:52 msg="completed recalculate owned streams job"
level=debug ts=2026-04-28T14:03:46.511559311Z caller=checkpoint.go:378 msg="writing series" size="132 kB"
level=warn ts=2026-04-28T14:03:47.639068187Z caller=ingester.go:497 component=ingester msg="autoforget have seen 0 unhealthy ingesters out of 1, network may be partioned, skip forgeting ingesters this round"